diff --git a/.gitattributes b/.gitattributes index 9a7c804b2d880bce3a4f5e8318d407385a219e3f..d24915691874c1bf94e1d55e820849de88106098 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text tokenizer.json filter=lfs diff=lfs merge=lfs -text tr13-176B-ml-t0-logs/logs/xp3capmixnewcodelonglossseq/main_log.txt filter=lfs diff=lfs merge=lfs -text tr13-176B-ml-t0-logs/logs/p31lossseq/main_log.txt filter=lfs diff=lfs merge=lfs -text +logs/logs/xp3capmixnewcodelonglossseq/main_log.txt filter=lfs diff=lfs merge=lfs -text +logs/logs/xp3zzlossseq/main_log.txt filter=lfs diff=lfs merge=lfs -text +logs/logs/p31lossseq/main_log.txt filter=lfs diff=lfs merge=lfs -text diff --git a/tr13-176B-ml-t0-logs/logs/p31lossseq/main_log.txt b/logs/logs/p31lossseq/main_log.txt similarity index 100% rename from tr13-176B-ml-t0-logs/logs/p31lossseq/main_log.txt rename to logs/logs/p31lossseq/main_log.txt diff --git a/tr13-176B-ml-t0-logs/logs/xp3capmixnewcodelonglossseq/main_log.txt b/logs/logs/xp3capmixnewcodelonglossseq/main_log.txt similarity index 100% rename from tr13-176B-ml-t0-logs/logs/xp3capmixnewcodelonglossseq/main_log.txt rename to logs/logs/xp3capmixnewcodelonglossseq/main_log.txt diff --git a/logs/logs/xp3zzlossseq/main_log.txt b/logs/logs/xp3zzlossseq/main_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..c81c9a2dbd193bc2a36c106e8320ab377c9af7c3 --- /dev/null +++ b/logs/logs/xp3zzlossseq/main_log.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fd97078e8a7ef82fc83d2529623fc05b5c796e41121a3133d63b9b581d8e631 +size 8553207 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662223716.jean-zay-iam52.1777228.0 b/logs/tensorboard/p31lossseq/events.out.tfevents.1662223716.jean-zay-iam52.1777228.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662223716.jean-zay-iam52.1777228.0 rename to logs/tensorboard/p31lossseq/events.out.tfevents.1662223716.jean-zay-iam52.1777228.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662224167.jean-zay-iam52.1778023.0 b/logs/tensorboard/p31lossseq/events.out.tfevents.1662224167.jean-zay-iam52.1778023.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662224167.jean-zay-iam52.1778023.0 rename to logs/tensorboard/p31lossseq/events.out.tfevents.1662224167.jean-zay-iam52.1778023.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662224257.jean-zay-iam52.1778680.0 b/logs/tensorboard/p31lossseq/events.out.tfevents.1662224257.jean-zay-iam52.1778680.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662224257.jean-zay-iam52.1778680.0 rename to logs/tensorboard/p31lossseq/events.out.tfevents.1662224257.jean-zay-iam52.1778680.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662225586.jean-zay-iam52.1780032.0 b/logs/tensorboard/p31lossseq/events.out.tfevents.1662225586.jean-zay-iam52.1780032.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662225586.jean-zay-iam52.1780032.0 rename to logs/tensorboard/p31lossseq/events.out.tfevents.1662225586.jean-zay-iam52.1780032.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662226813.jean-zay-iam52.1781537.0 b/logs/tensorboard/p31lossseq/events.out.tfevents.1662226813.jean-zay-iam52.1781537.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662226813.jean-zay-iam52.1781537.0 rename to logs/tensorboard/p31lossseq/events.out.tfevents.1662226813.jean-zay-iam52.1781537.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662227889.jean-zay-iam52.1786215.0 b/logs/tensorboard/p31lossseq/events.out.tfevents.1662227889.jean-zay-iam52.1786215.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseq/events.out.tfevents.1662227889.jean-zay-iam52.1786215.0 rename to logs/tensorboard/p31lossseq/events.out.tfevents.1662227889.jean-zay-iam52.1786215.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380144.jean-zay-iam52.1874785.0 b/logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380144.jean-zay-iam52.1874785.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380144.jean-zay-iam52.1874785.0 rename to logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380144.jean-zay-iam52.1874785.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380283.jean-zay-iam52.1875254.0 b/logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380283.jean-zay-iam52.1875254.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380283.jean-zay-iam52.1875254.0 rename to logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380283.jean-zay-iam52.1875254.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380488.jean-zay-iam52.1875989.0 b/logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380488.jean-zay-iam52.1875989.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380488.jean-zay-iam52.1875989.0 rename to logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380488.jean-zay-iam52.1875989.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380998.jean-zay-iam52.1880233.0 b/logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380998.jean-zay-iam52.1880233.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380998.jean-zay-iam52.1880233.0 rename to logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662380998.jean-zay-iam52.1880233.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662381259.jean-zay-iam52.1880987.0 b/logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662381259.jean-zay-iam52.1880987.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662381259.jean-zay-iam52.1880987.0 rename to logs/tensorboard/p31lossseqgs0/events.out.tfevents.1662381259.jean-zay-iam52.1880987.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661882858.jean-zay-iam52.1551482.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661882858.jean-zay-iam52.1551482.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661882858.jean-zay-iam52.1551482.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661882858.jean-zay-iam52.1551482.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661882970.jean-zay-iam52.1551905.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661882970.jean-zay-iam52.1551905.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661882970.jean-zay-iam52.1551905.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661882970.jean-zay-iam52.1551905.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661883090.jean-zay-iam52.1552322.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661883090.jean-zay-iam52.1552322.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661883090.jean-zay-iam52.1552322.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661883090.jean-zay-iam52.1552322.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661883576.jean-zay-iam52.1552955.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661883576.jean-zay-iam52.1552955.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661883576.jean-zay-iam52.1552955.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661883576.jean-zay-iam52.1552955.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884056.jean-zay-iam52.1553581.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884056.jean-zay-iam52.1553581.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884056.jean-zay-iam52.1553581.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884056.jean-zay-iam52.1553581.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884111.jean-zay-iam52.1553978.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884111.jean-zay-iam52.1553978.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884111.jean-zay-iam52.1553978.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884111.jean-zay-iam52.1553978.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884471.jean-zay-iam52.1554515.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884471.jean-zay-iam52.1554515.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884471.jean-zay-iam52.1554515.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884471.jean-zay-iam52.1554515.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884774.jean-zay-iam52.1555058.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884774.jean-zay-iam52.1555058.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884774.jean-zay-iam52.1555058.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884774.jean-zay-iam52.1555058.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884956.jean-zay-iam52.1555514.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884956.jean-zay-iam52.1555514.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884956.jean-zay-iam52.1555514.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661884956.jean-zay-iam52.1555514.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661885254.jean-zay-iam52.1556051.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661885254.jean-zay-iam52.1556051.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661885254.jean-zay-iam52.1556051.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661885254.jean-zay-iam52.1556051.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661885858.jean-zay-iam52.1556741.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661885858.jean-zay-iam52.1556741.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661885858.jean-zay-iam52.1556741.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661885858.jean-zay-iam52.1556741.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661886041.jean-zay-iam52.1557201.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661886041.jean-zay-iam52.1557201.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661886041.jean-zay-iam52.1557201.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661886041.jean-zay-iam52.1557201.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661887192.jean-zay-iam52.1558838.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661887192.jean-zay-iam52.1558838.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661887192.jean-zay-iam52.1558838.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661887192.jean-zay-iam52.1558838.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661887910.jean-zay-iam52.1560039.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661887910.jean-zay-iam52.1560039.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661887910.jean-zay-iam52.1560039.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661887910.jean-zay-iam52.1560039.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889058.jean-zay-iam52.1564747.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889058.jean-zay-iam52.1564747.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889058.jean-zay-iam52.1564747.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889058.jean-zay-iam52.1564747.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889172.jean-zay-iam52.1565381.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889172.jean-zay-iam52.1565381.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889172.jean-zay-iam52.1565381.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889172.jean-zay-iam52.1565381.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889295.jean-zay-iam52.1565797.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889295.jean-zay-iam52.1565797.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889295.jean-zay-iam52.1565797.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661889295.jean-zay-iam52.1565797.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890215.jean-zay-iam52.1570415.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890215.jean-zay-iam52.1570415.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890215.jean-zay-iam52.1570415.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890215.jean-zay-iam52.1570415.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890571.jean-zay-iam52.1571155.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890571.jean-zay-iam52.1571155.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890571.jean-zay-iam52.1571155.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890571.jean-zay-iam52.1571155.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890884.jean-zay-iam52.1571713.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890884.jean-zay-iam52.1571713.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890884.jean-zay-iam52.1571713.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661890884.jean-zay-iam52.1571713.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661893872.jean-zay-iam52.1577408.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661893872.jean-zay-iam52.1577408.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661893872.jean-zay-iam52.1577408.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1661893872.jean-zay-iam52.1577408.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662209529.jean-zay-iam47.881912.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662209529.jean-zay-iam47.881912.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662209529.jean-zay-iam47.881912.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662209529.jean-zay-iam47.881912.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662209688.jean-zay-iam47.882400.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662209688.jean-zay-iam47.882400.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662209688.jean-zay-iam47.882400.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662209688.jean-zay-iam47.882400.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662211160.jean-zay-iam47.883824.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662211160.jean-zay-iam47.883824.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662211160.jean-zay-iam47.883824.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662211160.jean-zay-iam47.883824.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662211596.jean-zay-iam47.884693.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662211596.jean-zay-iam47.884693.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662211596.jean-zay-iam47.884693.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662211596.jean-zay-iam47.884693.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662212436.jean-zay-iam19.1403782.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662212436.jean-zay-iam19.1403782.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662212436.jean-zay-iam19.1403782.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662212436.jean-zay-iam19.1403782.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662213274.jean-zay-iam47.889538.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662213274.jean-zay-iam47.889538.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662213274.jean-zay-iam47.889538.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662213274.jean-zay-iam47.889538.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662214552.jean-zay-iam47.894225.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662214552.jean-zay-iam47.894225.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662214552.jean-zay-iam47.894225.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662214552.jean-zay-iam47.894225.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662215541.jean-zay-iam47.898788.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662215541.jean-zay-iam47.898788.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662215541.jean-zay-iam47.898788.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662215541.jean-zay-iam47.898788.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662217247.jean-zay-iam47.903574.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662217247.jean-zay-iam47.903574.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662217247.jean-zay-iam47.903574.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662217247.jean-zay-iam47.903574.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662218547.jean-zay-iam47.908218.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662218547.jean-zay-iam47.908218.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662218547.jean-zay-iam47.908218.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662218547.jean-zay-iam47.908218.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662219507.jean-zay-iam47.912617.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662219507.jean-zay-iam47.912617.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662219507.jean-zay-iam47.912617.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662219507.jean-zay-iam47.912617.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662220634.jean-zay-iam52.1771424.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662220634.jean-zay-iam52.1771424.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662220634.jean-zay-iam52.1771424.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662220634.jean-zay-iam52.1771424.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662221027.jean-zay-iam47.921442.0 b/logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662221027.jean-zay-iam47.921442.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662221027.jean-zay-iam47.921442.0 rename to logs/tensorboard/xp3capmixnewcodelonglossseq/events.out.tfevents.1662221027.jean-zay-iam47.921442.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3zzlossseq/events.out.tfevents.1662580405.jean-zay-iam52.81767.0 b/logs/tensorboard/xp3zzlossseq/events.out.tfevents.1662580405.jean-zay-iam52.81767.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3zzlossseq/events.out.tfevents.1662580405.jean-zay-iam52.81767.0 rename to logs/tensorboard/xp3zzlossseq/events.out.tfevents.1662580405.jean-zay-iam52.81767.0 diff --git a/tr13-176B-ml-t0-logs/tensorboard/xp3zzlossseq/events.out.tfevents.1662581652.jean-zay-iam52.86551.0 b/logs/tensorboard/xp3zzlossseq/events.out.tfevents.1662581652.jean-zay-iam52.86551.0 similarity index 100% rename from tr13-176B-ml-t0-logs/tensorboard/xp3zzlossseq/events.out.tfevents.1662581652.jean-zay-iam52.86551.0 rename to logs/tensorboard/xp3zzlossseq/events.out.tfevents.1662581652.jean-zay-iam52.86551.0 diff --git a/tr13-176B-ml-t0-logs/logs/xp3zzlossseq/main_log.txt b/tr13-176B-ml-t0-logs/logs/xp3zzlossseq/main_log.txt deleted file mode 100644 index 4348013cdb4bdb9111783c15245d8cf457872af5..0000000000000000000000000000000000000000 --- a/tr13-176B-ml-t0-logs/logs/xp3zzlossseq/main_log.txt +++ /dev/null @@ -1,55084 +0,0 @@ -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]:Traceback (most recent call last): -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: pretrain( -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: args = parser.parse_args() -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]:Traceback (most recent call last): -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default4]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: args = parser.parse_args() -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default3]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default4]:Traceback (most recent call last): -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: main() -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: pretrain( -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: start_index = consume_optional(start_index) -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: args = parser.parse_args() -[default2]: with open(values, "r") as fi: -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default5]: return f(*args, **kwargs) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default7]:Traceback (most recent call last): -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default7]: return f(*args, **kwargs) -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default5]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: start_index = consume_optional(start_index) -[default6]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: return f(*args, **kwargs) -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default2]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: return f(*args, **kwargs) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default1]: args, argv = self.parse_known_args(args, namespace) -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]:Traceback (most recent call last): -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: pretrain( -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: take_action(action, args, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default1]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: args = parser.parse_args() -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default3]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default4]: pretrain( -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]:Traceback (most recent call last): -[default0]:Traceback (most recent call last): -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default0]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default0]:Traceback (most recent call last): -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: return f(*args, **kwargs) -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: main() -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default4]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default0]: args = parser.parse_args() -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]:Traceback (most recent call last): -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: args, argv = self.parse_known_args(args, namespace) -[default7]: pretrain( -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: args = parser.parse_args() -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]:Traceback (most recent call last): -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: args = parser.parse_args() -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]:Traceback (most recent call last): -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: pretrain( -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default5]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default3]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]:Traceback (most recent call last): -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: with open(values, "r") as fi: -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default1]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: start_index = consume_optional(start_index) -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]:Traceback (most recent call last): -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default4]: action(self, namespace, argument_values, option_string) -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default2]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default7]: main() -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: args = parser.parse_args() -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default7]: args = parser.parse_args() -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: take_action(action, args, option_string) -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]:Traceback (most recent call last): -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: main() -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: start_index = consume_optional(start_index) -[default7]: start_index = consume_optional(start_index) -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default0]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default2]: pretrain( -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default2]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default1]:Traceback (most recent call last): -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default3]: start_index = consume_optional(start_index) -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: take_action(action, args, option_string) -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: with open(values, "r") as fi: -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: action(self, namespace, argument_values, option_string) -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: return f(*args, **kwargs) -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default1]:Traceback (most recent call last): -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: main() -[default0]: pretrain( -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: pretrain( -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: with open(values, "r") as fi: -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default4]: start_index = consume_optional(start_index) -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: with open(values, "r") as fi: -[default4]: action(self, namespace, argument_values, option_string) -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: action(self, namespace, argument_values, option_string) -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default1]: pretrain( -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: take_action(action, args, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default1]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default0]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default0]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: main() -[default4]: start_index = consume_optional(start_index) -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default1]: with open(values, "r") as fi: -[default2]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: start_index = consume_optional(start_index) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: take_action(action, args, option_string) -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default6]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: with open(values, "r") as fi: -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: pretrain( -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: take_action(action, args, option_string) -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: start_index = consume_optional(start_index) -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]:Traceback (most recent call last): -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default4]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default3]: pretrain( -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: start_index = consume_optional(start_index) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: args = parser.parse_args() -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default1]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613322 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613323 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 113792 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 113796 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 97296 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 97298 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614189 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614196 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89325 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89332 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614214 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614216 closing signal SIGTERM -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 92446) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -[default0]:Traceback (most recent call last): -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default0]: main() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default0]: return f(*args, **kwargs) -[default0]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default0]: pretrain( -[default1]:Traceback (most recent call last): -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default1]: main() -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: initialize_megatron(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default1]: return f(*args, **kwargs) -[default1]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default1]: pretrain( -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default0]: set_global_variables(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default0]: args = _parse_args(extra_args_provider=extra_args_provider, -[default1]: initialize_megatron(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default0]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default1]: set_global_variables(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default1]: args = _parse_args(extra_args_provider=extra_args_provider, -[default0]: args = parser.parse_args() -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default1]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default0]: args, argv = self.parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: args = parser.parse_args() -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default1]: args, argv = self.parse_known_args(args, namespace) -[default0]: namespace, args = self._parse_known_args(args, namespace) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default1]: namespace, args = self._parse_known_args(args, namespace) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default0]: take_action(action, args, option_string) -[default0]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default1]: start_index = consume_optional(start_index) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default1]: take_action(action, args, option_string) -[default1]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default0]: action(self, namespace, argument_values, option_string) -[default0]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: action(self, namespace, argument_values, option_string) -[default1]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default1]: with open(values, "r") as fi: -[default0]: with open(values, "r") as fi: -[default0]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default1]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]:Traceback (most recent call last): -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: main() -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]:Traceback (most recent call last): -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default4]: return f(*args, **kwargs) -[default4]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: main() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default3]: return f(*args, **kwargs) -[default4]: pretrain( -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default3]: pretrain( -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default3]: initialize_megatron(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: initialize_megatron(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default4]: set_global_variables(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default4]: args = _parse_args(extra_args_provider=extra_args_provider, -[default3]: set_global_variables(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default3]: args = _parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default4]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: args = parser.parse_args() -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default3]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default4]: args, argv = self.parse_known_args(args, namespace) -[default3]: args = parser.parse_args() -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default3]: args, argv = self.parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default4]: namespace, args = self._parse_known_args(args, namespace) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: namespace, args = self._parse_known_args(args, namespace) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default3]: start_index = consume_optional(start_index) -[default4]: start_index = consume_optional(start_index) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default3]: take_action(action, args, option_string) -[default3]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default3]: action(self, namespace, argument_values, option_string) -[default3]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]: with open(values, "r") as fi: -[default4]: take_action(action, args, option_string) -[default4]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default4]: action(self, namespace, argument_values, option_string) -[default4]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default3]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default4]: with open(values, "r") as fi: -[default4]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default7]:Traceback (most recent call last): -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default7]: main() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default7]: return f(*args, **kwargs) -[default7]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default7]: pretrain( -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default7]: initialize_megatron(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default7]: set_global_variables(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default7]: args = _parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default7]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default7]: args = parser.parse_args() -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default7]: args, argv = self.parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default7]: namespace, args = self._parse_known_args(args, namespace) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default7]: start_index = consume_optional(start_index) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default7]: take_action(action, args, option_string) -[default7]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default7]: action(self, namespace, argument_values, option_string) -[default7]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default7]: with open(values, "r") as fi: -[default7]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default6]:Traceback (most recent call last): -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default6]: main() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default6]: return f(*args, **kwargs) -[default6]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default6]: pretrain( -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default6]: initialize_megatron(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default6]: set_global_variables(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default6]: args = _parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default6]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default6]: args = parser.parse_args() -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default6]: args, argv = self.parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default6]: namespace, args = self._parse_known_args(args, namespace) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default6]: start_index = consume_optional(start_index) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default6]: take_action(action, args, option_string) -[default6]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default6]: action(self, namespace, argument_values, option_string) -[default6]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default6]: with open(values, "r") as fi: -[default6]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 105530) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 613483) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 613465) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 2 (pid: 613324) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 81576) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 81850) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 86976) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 113789) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 81471) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 409595) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 97293) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 86537) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 614323) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 1 (pid: 614190) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 1 (pid: 89326) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 614368) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 614211) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 613525) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 106876) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 89524) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 86666) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 94493) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 86257) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 94492) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 86812) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 89352) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 106803) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 608918) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 164329) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 94460) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 123544) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 613914) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 94966) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -[default2]:Traceback (most recent call last): -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default2]: main() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default2]: return f(*args, **kwargs) -[default2]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default2]: pretrain( -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default2]: initialize_megatron(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default2]: set_global_variables(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default2]: args = _parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default2]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default2]: args = parser.parse_args() -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default2]: args, argv = self.parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default2]: namespace, args = self._parse_known_args(args, namespace) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default2]: start_index = consume_optional(start_index) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default2]: take_action(action, args, option_string) -[default2]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default2]: action(self, namespace, argument_values, option_string) -[default2]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default2]: with open(values, "r") as fi: -[default2]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -[default5]:Traceback (most recent call last): -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 207, in -[default5]: main() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -[default5]: return f(*args, **kwargs) -[default5]: File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main -[default5]: pretrain( -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain -[default5]: initialize_megatron(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron -[default5]: set_global_variables(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables -[default5]: args = _parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -[default5]: _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args -[default5]: args = parser.parse_args() -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -[default5]: args, argv = self.parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -[default5]: namespace, args = self._parse_known_args(args, namespace) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args -[default5]: start_index = consume_optional(start_index) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional -[default5]: take_action(action, args, option_string) -[default5]: File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -[default5]: action(self, namespace, argument_values, option_string) -[default5]: File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ -[default5]: with open(values, "r") as fi: -[default5]:FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 93649) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 101449) of binary: /gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/bin/python -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - return _run_code(code, main_globals, None, - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - exec(code, run_globals) -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - return _run_code(code, main_globals, None, - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - raise ChildFailedError( - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - main() -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam45-ib0 - rank : 241 (local_rank: 1) - exitcode : 1 (pid: 614324) - error_file: /tmp/torchelastic_y9494pza/none_5gfgg3at/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam45-ib0 - rank : 242 (local_rank: 2) - exitcode : 1 (pid: 614325) - error_file: /tmp/torchelastic_y9494pza/none_5gfgg3at/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam45-ib0 - rank : 243 (local_rank: 3) - exitcode : 1 (pid: 614326) - error_file: /tmp/torchelastic_y9494pza/none_5gfgg3at/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - run(args) -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam45-ib0 - rank : 244 (local_rank: 4) - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - exitcode : 1 (pid: 614327) - error_file: /tmp/torchelastic_y9494pza/none_5gfgg3at/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam45-ib0 - rank : 245 (local_rank: 5) - exitcode : 1 (pid: 614328) - error_file: /tmp/torchelastic_y9494pza/none_5gfgg3at/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam45-ib0 - rank : 246 (local_rank: 6) - exitcode : 1 (pid: 614329) - error_file: /tmp/torchelastic_y9494pza/none_5gfgg3at/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam45-ib0 - rank : 247 (local_rank: 7) - exitcode : 1 (pid: 614330) - error_file: /tmp/torchelastic_y9494pza/none_5gfgg3at/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam45-ib0 - rank : 240 (local_rank: 0) - exitcode : 1 (pid: 614323) - error_file: /tmp/torchelastic_y9494pza/none_5gfgg3at/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - elastic_launch( - raise ChildFailedError( - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam44-ib0 - rank : 233 (local_rank: 1) - exitcode : 1 (pid: 613484) - error_file: /tmp/torchelastic_6ewpocrd/none_riyse4wl/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - return _run_code(code, main_globals, None, - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam44-ib0 - rank : 234 (local_rank: 2) - exitcode : 1 (pid: 613485) - error_file: /tmp/torchelastic_6ewpocrd/none_riyse4wl/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - return _run_code(code, main_globals, None, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - exec(code, run_globals) - raise ChildFailedError( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam44-ib0 - rank : 235 (local_rank: 3) - exitcode : 1 (pid: 613486) - error_file: /tmp/torchelastic_6ewpocrd/none_riyse4wl/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - return _run_code(code, main_globals, None, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam44-ib0 - rank : 236 (local_rank: 4) -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam21-ib0 - rank : 121 (local_rank: 1) - exitcode : 1 (pid: 86258) - error_file: /tmp/torchelastic_f6e_dfom/none_bdzg4i52/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - exitcode : 1 (pid: 613487) - error_file: /tmp/torchelastic_6ewpocrd/none_riyse4wl/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam44-ib0 - rank : 237 (local_rank: 5) - exitcode : 1 (pid: 613488) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - error_file: /tmp/torchelastic_6ewpocrd/none_riyse4wl/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam44-ib0 - rank : 238 (local_rank: 6) - exitcode : 1 (pid: 613489) - error_file: /tmp/torchelastic_6ewpocrd/none_riyse4wl/attempt_0/6/error.json - traceback : Traceback (most recent call last): - return _run_code(code, main_globals, None, - exec(code, run_globals) -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam44-ib0 - rank : 239 (local_rank: 7) - exitcode : 1 (pid: 613490) - error_file: /tmp/torchelastic_6ewpocrd/none_riyse4wl/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam21-ib0 - rank : 122 (local_rank: 2) - exitcode : 1 (pid: 86259) - error_file: /tmp/torchelastic_f6e_dfom/none_bdzg4i52/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - return _run_code(code, main_globals, None, -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - exec(code, run_globals) - main() - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam21-ib0 - rank : 123 (local_rank: 3) - exitcode : 1 (pid: 86260) - error_file: /tmp/torchelastic_f6e_dfom/none_bdzg4i52/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam21-ib0 - rank : 124 (local_rank: 4) - return _run_code(code, main_globals, None, - exitcode : 1 (pid: 86261) - error_file: /tmp/torchelastic_f6e_dfom/none_bdzg4i52/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam21-ib0 - rank : 125 (local_rank: 5) - exitcode : 1 (pid: 86262) - error_file: /tmp/torchelastic_f6e_dfom/none_bdzg4i52/attempt_0/5/error.json - traceback : Traceback (most recent call last): - return f(*args, **kwargs) -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - exec(code, run_globals) - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam21-ib0 - rank : 126 (local_rank: 6) - exitcode : 1 (pid: 86263) - error_file: /tmp/torchelastic_f6e_dfom/none_bdzg4i52/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - exec(code, run_globals) - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - main() - main() - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - main() - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam21-ib0 - rank : 127 (local_rank: 7) - exitcode : 1 (pid: 86264) - error_file: /tmp/torchelastic_f6e_dfom/none_bdzg4i52/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - main() - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam21-ib0 - rank : 120 (local_rank: 0) - exitcode : 1 (pid: 86257) - error_file: /tmp/torchelastic_f6e_dfom/none_bdzg4i52/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - exec(code, run_globals) - return f(*args, **kwargs) - return f(*args, **kwargs) - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return f(*args, **kwargs) - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - run(args) - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - return f(*args, **kwargs) - run(args) -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - run(args) -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - raise ChildFailedError( - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - elastic_launch( - raise ChildFailedError( - elastic_launch( - return _run_code(code, main_globals, None, - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - run(args) - run(args) - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam18-ib0 - rank : 97 (local_rank: 1) - exitcode : 1 (pid: 86813) - error_file: /tmp/torchelastic_jodlwqrn/none_uunipkh9/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - return launch_agent(self._config, self._entrypoint, list(args)) -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam39-ib0 - rank : 195 (local_rank: 3) - exitcode : 1 (pid: 613325) - error_file: /tmp/torchelastic_ex_vj8ld/none_4sfl6n9l/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - exec(code, run_globals) - exec(code, run_globals) - raise ChildFailedError( - raise ChildFailedError( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam18-ib0 - rank : 98 (local_rank: 2) - exitcode : 1 (pid: 86814) - error_file: /tmp/torchelastic_jodlwqrn/none_uunipkh9/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - raise ChildFailedError( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam39-ib0 - rank : 196 (local_rank: 4) - exitcode : 1 (pid: 613326) - error_file: /tmp/torchelastic_ex_vj8ld/none_4sfl6n9l/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam05-ib0 - rank : 17 (local_rank: 1) - exitcode : 1 (pid: 89353) - error_file: /tmp/torchelastic_yvfwl4hg/none_efpnja32/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - elastic_launch( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam42-ib0 - rank : 217 (local_rank: 1) - exitcode : 1 (pid: 614212) - error_file: /tmp/torchelastic_2l5pjca2/none_yd9mc1in/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - raise ChildFailedError( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam36-ib0 - rank : 169 (local_rank: 1) - exitcode : 1 (pid: 614369) - error_file: /tmp/torchelastic_980i2bc8/none_drv1bl08/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam18-ib0 - rank : 99 (local_rank: 3) - exitcode : 1 (pid: 86815) - error_file: /tmp/torchelastic_jodlwqrn/none_uunipkh9/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - return _run_code(code, main_globals, None, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam39-ib0 - rank : 197 (local_rank: 5) - exitcode : 1 (pid: 613327) - error_file: /tmp/torchelastic_ex_vj8ld/none_4sfl6n9l/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - main() - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam08-ib0 - rank : 41 (local_rank: 1) - exitcode : 1 (pid: 94461) - error_file: /tmp/torchelastic_zq2deldf/none_dpd7oh5o/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam18-ib0 - rank : 100 (local_rank: 4) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam42-ib0 - rank : 218 (local_rank: 2) - exitcode : 1 (pid: 614213) - error_file: /tmp/torchelastic_2l5pjca2/none_yd9mc1in/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam39-ib0 - rank : 198 (local_rank: 6) - exitcode : 1 (pid: 86816) - error_file: /tmp/torchelastic_jodlwqrn/none_uunipkh9/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam36-ib0 - rank : 170 (local_rank: 2) - exitcode : 1 (pid: 614370) - error_file: /tmp/torchelastic_980i2bc8/none_drv1bl08/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - elastic_launch( - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam05-ib0 - rank : 18 (local_rank: 2) - exitcode : 1 (pid: 89354) - error_file: /tmp/torchelastic_yvfwl4hg/none_efpnja32/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - elastic_launch( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - exitcode : 1 (pid: 613328) - error_file: /tmp/torchelastic_ex_vj8ld/none_4sfl6n9l/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - return launch_agent(self._config, self._entrypoint, list(args)) -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam18-ib0 - rank : 101 (local_rank: 5) - exitcode : 1 (pid: 86817) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam39-ib0 - rank : 199 (local_rank: 7) - exitcode : 1 (pid: 613329) - error_file: /tmp/torchelastic_jodlwqrn/none_uunipkh9/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - return launch_agent(self._config, self._entrypoint, list(args)) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam42-ib0 - rank : 220 (local_rank: 4) - exitcode : 1 (pid: 614215) - error_file: /tmp/torchelastic_2l5pjca2/none_yd9mc1in/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam08-ib0 - rank : 42 (local_rank: 2) - exitcode : 1 (pid: 94462) - error_file: /tmp/torchelastic_zq2deldf/none_dpd7oh5o/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - error_file: /tmp/torchelastic_ex_vj8ld/none_4sfl6n9l/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - main() - main() -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return f(*args, **kwargs) - return launch_agent(self._config, self._entrypoint, list(args)) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam05-ib0 - rank : 19 (local_rank: 3) - exitcode : 1 (pid: 89355) - error_file: /tmp/torchelastic_yvfwl4hg/none_efpnja32/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam36-ib0 - rank : 171 (local_rank: 3) - exitcode : 1 (pid: 614371) - error_file: /tmp/torchelastic_980i2bc8/none_drv1bl08/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam18-ib0 - rank : 102 (local_rank: 6) - exitcode : 1 (pid: 86818) - error_file: /tmp/torchelastic_jodlwqrn/none_uunipkh9/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam42-ib0 - rank : 222 (local_rank: 6) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam39-ib0 - rank : 194 (local_rank: 2) - exitcode : 1 (pid: 613324) - error_file: /tmp/torchelastic_ex_vj8ld/none_4sfl6n9l/attempt_0/2/error.json - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam05-ib0 - rank : 20 (local_rank: 4) - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - raise ChildFailedError( - exitcode : 1 (pid: 614217) - error_file: /tmp/torchelastic_2l5pjca2/none_yd9mc1in/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam08-ib0 - rank : 43 (local_rank: 3) - exitcode : 1 (pid: 94463) - error_file: /tmp/torchelastic_zq2deldf/none_dpd7oh5o/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam36-ib0 - rank : 172 (local_rank: 4) - main() - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - raise ChildFailedError( - exitcode : 1 (pid: 89356) - error_file: /tmp/torchelastic_yvfwl4hg/none_efpnja32/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - raise ChildFailedError( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - exitcode : 1 (pid: 614372) - error_file: /tmp/torchelastic_980i2bc8/none_drv1bl08/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - return f(*args, **kwargs) - return _run_code(code, main_globals, None, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam42-ib0 - rank : 223 (local_rank: 7) - exitcode : 1 (pid: 614218) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam18-ib0 - rank : 103 (local_rank: 7) - exitcode : 1 (pid: 86819) - error_file: /tmp/torchelastic_jodlwqrn/none_uunipkh9/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam05-ib0 - rank : 21 (local_rank: 5) - exitcode : 1 (pid: 89357) -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam19-ib0 - rank : 105 (local_rank: 1) - exitcode : 1 (pid: 81577) - error_file: /tmp/torchelastic_h4egvawr/none_9w7b7ic7/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - error_file: /tmp/torchelastic_2l5pjca2/none_yd9mc1in/attempt_0/7/error.json - traceback : Traceback (most recent call last): - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam08-ib0 - rank : 44 (local_rank: 4) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam36-ib0 - rank : 173 (local_rank: 5) - exitcode : 1 (pid: 614373) -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam38-ib0 - rank : 185 (local_rank: 1) - exitcode : 1 (pid: 613915) - error_file: /tmp/torchelastic_5ouh2e19/none_e1a44elt/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - return _run_code(code, main_globals, None, - main() - error_file: /tmp/torchelastic_yvfwl4hg/none_efpnja32/attempt_0/5/error.json - traceback : Traceback (most recent call last): -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam03-ib0 - rank : 1 (local_rank: 1) - exitcode : 1 (pid: 94967) - error_file: /tmp/torchelastic_4bf22it8/none_ydmoaq14/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - main() - exitcode : 1 (pid: 94464) - error_file: /tmp/torchelastic_zq2deldf/none_dpd7oh5o/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - error_file: /tmp/torchelastic_980i2bc8/none_drv1bl08/attempt_0/5/error.json - traceback : Traceback (most recent call last): - return f(*args, **kwargs) - main() - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - run(args) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam19-ib0 - rank : 106 (local_rank: 2) - exitcode : 1 (pid: 81578) - error_file: /tmp/torchelastic_h4egvawr/none_9w7b7ic7/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam42-ib0 - rank : 216 (local_rank: 0) - exitcode : 1 (pid: 614211) - error_file: /tmp/torchelastic_2l5pjca2/none_yd9mc1in/attempt_0/0/error.json - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam08-ib0 - rank : 45 (local_rank: 5) - exitcode : 1 (pid: 94465) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam18-ib0 - rank : 96 (local_rank: 0) - exitcode : 1 (pid: 86812) - error_file: /tmp/torchelastic_jodlwqrn/none_uunipkh9/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam38-ib0 - rank : 186 (local_rank: 2) - exitcode : 1 (pid: 613916) - error_file: /tmp/torchelastic_5ouh2e19/none_e1a44elt/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - exec(code, run_globals) - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam05-ib0 - rank : 22 (local_rank: 6) - exitcode : 1 (pid: 89358) - error_file: /tmp/torchelastic_yvfwl4hg/none_efpnja32/attempt_0/6/error.json - traceback : Traceback (most recent call last): - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam03-ib0 - rank : 2 (local_rank: 2) - exitcode : 1 (pid: 94968) - error_file: /tmp/torchelastic_4bf22it8/none_ydmoaq14/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - return f(*args, **kwargs) - error_file: /tmp/torchelastic_zq2deldf/none_dpd7oh5o/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam36-ib0 - rank : 174 (local_rank: 6) - exitcode : 1 (pid: 614374) - error_file: /tmp/torchelastic_980i2bc8/none_drv1bl08/attempt_0/6/error.json - traceback : Traceback (most recent call last): - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - run(args) - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - run(args) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam19-ib0 - rank : 107 (local_rank: 3) - exitcode : 1 (pid: 81579) - error_file: /tmp/torchelastic_h4egvawr/none_9w7b7ic7/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam08-ib0 - rank : 46 (local_rank: 6) - exitcode : 1 (pid: 94466) - error_file: /tmp/torchelastic_zq2deldf/none_dpd7oh5o/attempt_0/6/error.json - traceback : Traceback (most recent call last): - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam38-ib0 - rank : 187 (local_rank: 3) - exitcode : 1 (pid: 613917) - error_file: /tmp/torchelastic_5ouh2e19/none_e1a44elt/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam05-ib0 - rank : 23 (local_rank: 7) - exitcode : 1 (pid: 89359) - error_file: /tmp/torchelastic_yvfwl4hg/none_efpnja32/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam03-ib0 - rank : 3 (local_rank: 3) - exitcode : 1 (pid: 94969) - error_file: /tmp/torchelastic_4bf22it8/none_ydmoaq14/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam36-ib0 - rank : 175 (local_rank: 7) - exitcode : 1 (pid: 614375) - error_file: /tmp/torchelastic_980i2bc8/none_drv1bl08/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - run(args) - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - elastic_launch( - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam19-ib0 - rank : 108 (local_rank: 4) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam38-ib0 - rank : 188 (local_rank: 4) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam03-ib0 - rank : 4 (local_rank: 4) - exitcode : 1 (pid: 81580) - error_file: /tmp/torchelastic_h4egvawr/none_9w7b7ic7/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam08-ib0 - rank : 47 (local_rank: 7) - exitcode : 1 (pid: 94467) - error_file: /tmp/torchelastic_zq2deldf/none_dpd7oh5o/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - exitcode : 1 (pid: 613918) - error_file: /tmp/torchelastic_5ouh2e19/none_e1a44elt/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam05-ib0 - rank : 16 (local_rank: 0) - exitcode : 1 (pid: 89352) - error_file: /tmp/torchelastic_yvfwl4hg/none_efpnja32/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - exitcode : 1 (pid: 94970) - error_file: /tmp/torchelastic_4bf22it8/none_ydmoaq14/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam36-ib0 - rank : 168 (local_rank: 0) - exitcode : 1 (pid: 614368) - error_file: /tmp/torchelastic_980i2bc8/none_drv1bl08/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam19-ib0 - rank : 109 (local_rank: 5) - exitcode : 1 (pid: 81581) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam38-ib0 - rank : 189 (local_rank: 5) - exitcode : 1 (pid: 613919) - main() - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam03-ib0 - rank : 5 (local_rank: 5) - exitcode : 1 (pid: 94971) - error_file: /tmp/torchelastic_h4egvawr/none_9w7b7ic7/attempt_0/5/error.json - traceback : Traceback (most recent call last): - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - return launch_agent(self._config, self._entrypoint, list(args)) - error_file: /tmp/torchelastic_5ouh2e19/none_e1a44elt/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - error_file: /tmp/torchelastic_4bf22it8/none_ydmoaq14/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - return _run_code(code, main_globals, None, - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - run(args) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam08-ib0 - rank : 40 (local_rank: 0) - exitcode : 1 (pid: 94460) - error_file: /tmp/torchelastic_zq2deldf/none_dpd7oh5o/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam19-ib0 - rank : 110 (local_rank: 6) - exitcode : 1 (pid: 81582) - error_file: /tmp/torchelastic_h4egvawr/none_9w7b7ic7/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - elastic_launch( - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam38-ib0 - rank : 190 (local_rank: 6) - exitcode : 1 (pid: 613920) - error_file: /tmp/torchelastic_5ouh2e19/none_e1a44elt/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam03-ib0 - rank : 6 (local_rank: 6) - exitcode : 1 (pid: 94972) - error_file: /tmp/torchelastic_4bf22it8/none_ydmoaq14/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - raise ChildFailedError( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - elastic_launch( -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - return f(*args, **kwargs) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam19-ib0 - rank : 111 (local_rank: 7) - exitcode : 1 (pid: 81583) - error_file: /tmp/torchelastic_h4egvawr/none_9w7b7ic7/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam38-ib0 - rank : 191 (local_rank: 7) - exitcode : 1 (pid: 613921) - error_file: /tmp/torchelastic_5ouh2e19/none_e1a44elt/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam03-ib0 - rank : 7 (local_rank: 7) - exitcode : 1 (pid: 94973) - error_file: /tmp/torchelastic_4bf22it8/none_ydmoaq14/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam51-ib0 - rank : 273 (local_rank: 1) - exitcode : 1 (pid: 86667) - error_file: /tmp/torchelastic_t26_afso/none_5vf2qvzh/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam19-ib0 - rank : 104 (local_rank: 0) - exitcode : 1 (pid: 81576) - error_file: /tmp/torchelastic_h4egvawr/none_9w7b7ic7/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - raise ChildFailedError( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam38-ib0 - rank : 184 (local_rank: 0) - exitcode : 1 (pid: 613914) - error_file: /tmp/torchelastic_5ouh2e19/none_e1a44elt/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam03-ib0 - rank : 0 (local_rank: 0) - exitcode : 1 (pid: 94966) - error_file: /tmp/torchelastic_4bf22it8/none_ydmoaq14/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - return launch_agent(self._config, self._entrypoint, list(args)) - return _run_code(code, main_globals, None, -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam50-ib0 - rank : 265 (local_rank: 1) - exitcode : 1 (pid: 81851) - error_file: /tmp/torchelastic_13qo8ed7/none_wrvi8qx_/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam51-ib0 - rank : 274 (local_rank: 2) - exitcode : 1 (pid: 86668) - error_file: /tmp/torchelastic_t26_afso/none_5vf2qvzh/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - elastic_launch( - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam51-ib0 - rank : 275 (local_rank: 3) - exitcode : 1 (pid: 86669) - error_file: /tmp/torchelastic_t26_afso/none_5vf2qvzh/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - raise ChildFailedError( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - exec(code, run_globals) - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam50-ib0 - rank : 266 (local_rank: 2) - exitcode : 1 (pid: 81853) - error_file: /tmp/torchelastic_13qo8ed7/none_wrvi8qx_/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam51-ib0 - rank : 276 (local_rank: 4) - run(args) - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - exitcode : 1 (pid: 86670) - error_file: /tmp/torchelastic_t26_afso/none_5vf2qvzh/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam46-ib0 - rank : 249 (local_rank: 1) - exitcode : 1 (pid: 613466) - error_file: /tmp/torchelastic_8zghg4n3/none_i2taw_uh/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam51-ib0 - rank : 277 (local_rank: 5) - exitcode : 1 (pid: 86671) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam50-ib0 - rank : 267 (local_rank: 3) - exitcode : 1 (pid: 81854) - error_file: /tmp/torchelastic_13qo8ed7/none_wrvi8qx_/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - error_file: /tmp/torchelastic_t26_afso/none_5vf2qvzh/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam50-ib0 - rank : 268 (local_rank: 4) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam51-ib0 - rank : 278 (local_rank: 6) - exitcode : 1 (pid: 86672) - error_file: /tmp/torchelastic_t26_afso/none_5vf2qvzh/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam46-ib0 - rank : 250 (local_rank: 2) - exitcode : 1 (pid: 613467) - error_file: /tmp/torchelastic_8zghg4n3/none_i2taw_uh/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - exitcode : 1 (pid: 81855) - error_file: /tmp/torchelastic_13qo8ed7/none_wrvi8qx_/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam50-ib0 - rank : 269 (local_rank: 5) - exitcode : 1 (pid: 81856) - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - return _run_code(code, main_globals, None, - error_file: /tmp/torchelastic_13qo8ed7/none_wrvi8qx_/attempt_0/5/error.json - traceback : Traceback (most recent call last): - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam46-ib0 - rank : 251 (local_rank: 3) - exitcode : 1 (pid: 613468) - error_file: /tmp/torchelastic_8zghg4n3/none_i2taw_uh/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam51-ib0 - rank : 279 (local_rank: 7) - exitcode : 1 (pid: 86673) - error_file: /tmp/torchelastic_t26_afso/none_5vf2qvzh/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - elastic_launch( - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam50-ib0 - rank : 270 (local_rank: 6) - exitcode : 1 (pid: 81857) - error_file: /tmp/torchelastic_13qo8ed7/none_wrvi8qx_/attempt_0/6/error.json - traceback : Traceback (most recent call last): - exec(code, run_globals) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam46-ib0 - rank : 252 (local_rank: 4) - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - exitcode : 1 (pid: 613469) - error_file: /tmp/torchelastic_8zghg4n3/none_i2taw_uh/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam51-ib0 - rank : 272 (local_rank: 0) - exitcode : 1 (pid: 86666) - error_file: /tmp/torchelastic_t26_afso/none_5vf2qvzh/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam46-ib0 - rank : 253 (local_rank: 5) - exitcode : 1 (pid: 613470) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam50-ib0 - rank : 271 (local_rank: 7) - exitcode : 1 (pid: 81858) - error_file: /tmp/torchelastic_13qo8ed7/none_wrvi8qx_/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - raise ChildFailedError( - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - error_file: /tmp/torchelastic_8zghg4n3/none_i2taw_uh/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam46-ib0 - rank : 254 (local_rank: 6) - exitcode : 1 (pid: 613471) - error_file: /tmp/torchelastic_8zghg4n3/none_i2taw_uh/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam50-ib0 - rank : 264 (local_rank: 0) - exitcode : 1 (pid: 81850) - error_file: /tmp/torchelastic_13qo8ed7/none_wrvi8qx_/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - raise ChildFailedError( - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam43-ib0 - rank : 225 (local_rank: 1) - exitcode : 1 (pid: 608919) - error_file: /tmp/torchelastic_q4e6czlf/none_p96ivmsa/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam46-ib0 - rank : 255 (local_rank: 7) - exitcode : 1 (pid: 613472) - error_file: /tmp/torchelastic_8zghg4n3/none_i2taw_uh/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam10-ib0 - rank : 57 (local_rank: 1) - exitcode : 1 (pid: 409596) - error_file: /tmp/torchelastic_aoag9upe/none_ssn8gpl0/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam46-ib0 - rank : 248 (local_rank: 0) - exitcode : 1 (pid: 613465) - error_file: /tmp/torchelastic_8zghg4n3/none_i2taw_uh/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam43-ib0 - rank : 226 (local_rank: 2) - exitcode : 1 (pid: 608920) - error_file: /tmp/torchelastic_q4e6czlf/none_p96ivmsa/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - main() - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam10-ib0 - rank : 58 (local_rank: 2) - exitcode : 1 (pid: 409597) - error_file: /tmp/torchelastic_aoag9upe/none_ssn8gpl0/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - return _run_code(code, main_globals, None, - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam43-ib0 - rank : 227 (local_rank: 3) - exitcode : 1 (pid: 608921) - error_file: /tmp/torchelastic_q4e6czlf/none_p96ivmsa/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam43-ib0 - rank : 228 (local_rank: 4) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam10-ib0 - rank : 59 (local_rank: 3) - exitcode : 1 (pid: 409598) - error_file: /tmp/torchelastic_aoag9upe/none_ssn8gpl0/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - exitcode : 1 (pid: 608922) - error_file: /tmp/torchelastic_q4e6czlf/none_p96ivmsa/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam43-ib0 - rank : 229 (local_rank: 5) - exitcode : 1 (pid: 608923) - return f(*args, **kwargs) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam10-ib0 - rank : 60 (local_rank: 4) - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - error_file: /tmp/torchelastic_q4e6czlf/none_p96ivmsa/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - exitcode : 1 (pid: 409599) - error_file: /tmp/torchelastic_aoag9upe/none_ssn8gpl0/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - main() - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam10-ib0 - rank : 61 (local_rank: 5) - exitcode : 1 (pid: 409600) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam43-ib0 - rank : 230 (local_rank: 6) - exitcode : 1 (pid: 608924) - error_file: /tmp/torchelastic_q4e6czlf/none_p96ivmsa/attempt_0/6/error.json - traceback : Traceback (most recent call last): - error_file: /tmp/torchelastic_aoag9upe/none_ssn8gpl0/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam10-ib0 - rank : 62 (local_rank: 6) - exitcode : 1 (pid: 409601) - error_file: /tmp/torchelastic_aoag9upe/none_ssn8gpl0/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam43-ib0 - rank : 231 (local_rank: 7) - exitcode : 1 (pid: 608925) - error_file: /tmp/torchelastic_q4e6czlf/none_p96ivmsa/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam10-ib0 - rank : 63 (local_rank: 7) - exitcode : 1 (pid: 409602) - error_file: /tmp/torchelastic_aoag9upe/none_ssn8gpl0/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam43-ib0 - rank : 224 (local_rank: 0) - exitcode : 1 (pid: 608918) - error_file: /tmp/torchelastic_q4e6czlf/none_p96ivmsa/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam10-ib0 - rank : 56 (local_rank: 0) - exitcode : 1 (pid: 409595) - error_file: /tmp/torchelastic_aoag9upe/none_ssn8gpl0/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - run(args) - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - return launch_agent(self._config, self._entrypoint, list(args)) - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - raise ChildFailedError( - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - raise ChildFailedError( - raise ChildFailedError( - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam20-ib0 - rank : 113 (local_rank: 1) - exitcode : 1 (pid: 86538) - error_file: /tmp/torchelastic_cdzsak1n/none_k7ljn2kh/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - return launch_agent(self._config, self._entrypoint, list(args)) -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam25-ib0 - rank : 138 (local_rank: 2) - exitcode : 1 (pid: 89327) - error_file: /tmp/torchelastic_jza_sak3/none_7po77vm2/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam52-ib0 - rank : 281 (local_rank: 1) - exitcode : 1 (pid: 81472) - error_file: /tmp/torchelastic_vw2jgzlb/none_3_1r0v7g/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - raise ChildFailedError( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam04-ib0 - rank : 9 (local_rank: 1) - exitcode : 1 (pid: 94493) - error_file: /tmp/torchelastic__z99v7jd/none_wzje8khg/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - raise ChildFailedError( - raise ChildFailedError( - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam25-ib0 - rank : 139 (local_rank: 3) - exitcode : 1 (pid: 89328) - error_file: /tmp/torchelastic_jza_sak3/none_7po77vm2/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam52-ib0 - rank : 282 (local_rank: 2) - exitcode : 1 (pid: 81473) - error_file: /tmp/torchelastic_vw2jgzlb/none_3_1r0v7g/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam20-ib0 - rank : 114 (local_rank: 2) - exitcode : 1 (pid: 86539) - error_file: /tmp/torchelastic_cdzsak1n/none_k7ljn2kh/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam15-ib0 - rank : 89 (local_rank: 1) - exitcode : 1 (pid: 92447) - error_file: /tmp/torchelastic_xajhzjf7/none_xnzgml15/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam09-ib0 - rank : 49 (local_rank: 1) - exitcode : 1 (pid: 89525) - error_file: /tmp/torchelastic_qj1as6wk/none_82do5wit/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - raise ChildFailedError( - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam04-ib0 - rank : 10 (local_rank: 2) - exitcode : 1 (pid: 94494) - error_file: /tmp/torchelastic__z99v7jd/none_wzje8khg/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam25-ib0 - rank : 140 (local_rank: 4) - exitcode : 1 (pid: 89329) - error_file: /tmp/torchelastic_jza_sak3/none_7po77vm2/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam52-ib0 - rank : 283 (local_rank: 3) - exitcode : 1 (pid: 81474) - error_file: /tmp/torchelastic_vw2jgzlb/none_3_1r0v7g/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam20-ib0 - rank : 115 (local_rank: 3) - exitcode : 1 (pid: 86540) - error_file: /tmp/torchelastic_cdzsak1n/none_k7ljn2kh/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam09-ib0 - rank : 50 (local_rank: 2) - exitcode : 1 (pid: 89526) - error_file: /tmp/torchelastic_qj1as6wk/none_82do5wit/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - main() - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam35-ib0 - rank : 161 (local_rank: 1) - exitcode : 1 (pid: 164330) - error_file: /tmp/torchelastic_qxt35ay2/none_a9dp3k0h/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam25-ib0 - rank : 141 (local_rank: 5) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam52-ib0 - rank : 284 (local_rank: 4) - main() - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam20-ib0 - rank : 116 (local_rank: 4) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam15-ib0 - rank : 90 (local_rank: 2) - exitcode : 1 (pid: 92448) - error_file: /tmp/torchelastic_xajhzjf7/none_xnzgml15/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - exitcode : 1 (pid: 89330) - error_file: /tmp/torchelastic_jza_sak3/none_7po77vm2/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - exitcode : 1 (pid: 81475) - error_file: /tmp/torchelastic_vw2jgzlb/none_3_1r0v7g/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - exitcode : 1 (pid: 86541) - error_file: /tmp/torchelastic_cdzsak1n/none_k7ljn2kh/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam04-ib0 - rank : 11 (local_rank: 3) - exitcode : 1 (pid: 94495) - error_file: /tmp/torchelastic__z99v7jd/none_wzje8khg/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam09-ib0 - rank : 51 (local_rank: 3) - exitcode : 1 (pid: 89527) - error_file: /tmp/torchelastic_qj1as6wk/none_82do5wit/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam25-ib0 - rank : 142 (local_rank: 6) - exitcode : 1 (pid: 89331) - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam52-ib0 - rank : 285 (local_rank: 5) - exitcode : 1 (pid: 81476) - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam35-ib0 - rank : 162 (local_rank: 2) - exitcode : 1 (pid: 164331) - error_file: /tmp/torchelastic_qxt35ay2/none_a9dp3k0h/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam20-ib0 - rank : 117 (local_rank: 5) - exitcode : 1 (pid: 86542) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - error_file: /tmp/torchelastic_jza_sak3/none_7po77vm2/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - error_file: /tmp/torchelastic_vw2jgzlb/none_3_1r0v7g/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - error_file: /tmp/torchelastic_cdzsak1n/none_k7ljn2kh/attempt_0/5/error.json - traceback : Traceback (most recent call last): - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam04-ib0 - rank : 12 (local_rank: 4) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam15-ib0 - rank : 91 (local_rank: 3) - exitcode : 1 (pid: 92449) - error_file: /tmp/torchelastic_xajhzjf7/none_xnzgml15/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - exitcode : 1 (pid: 94496) - error_file: /tmp/torchelastic__z99v7jd/none_wzje8khg/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam09-ib0 - rank : 52 (local_rank: 4) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - return f(*args, **kwargs) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - exitcode : 1 (pid: 89528) - error_file: /tmp/torchelastic_qj1as6wk/none_82do5wit/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam25-ib0 - rank : 137 (local_rank: 1) - exitcode : 1 (pid: 89326) - error_file: /tmp/torchelastic_jza_sak3/none_7po77vm2/attempt_0/1/error.json - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam52-ib0 - rank : 286 (local_rank: 6) - exitcode : 1 (pid: 81477) - error_file: /tmp/torchelastic_vw2jgzlb/none_3_1r0v7g/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam35-ib0 - rank : 163 (local_rank: 3) - exitcode : 1 (pid: 164332) - error_file: /tmp/torchelastic_qxt35ay2/none_a9dp3k0h/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam20-ib0 - rank : 118 (local_rank: 6) - exitcode : 1 (pid: 86543) - error_file: /tmp/torchelastic_cdzsak1n/none_k7ljn2kh/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam04-ib0 - rank : 13 (local_rank: 5) - exitcode : 1 (pid: 94497) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam15-ib0 - rank : 92 (local_rank: 4) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - error_file: /tmp/torchelastic__z99v7jd/none_wzje8khg/attempt_0/5/error.json - traceback : Traceback (most recent call last): - exitcode : 1 (pid: 92450) - error_file: /tmp/torchelastic_xajhzjf7/none_xnzgml15/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam09-ib0 - rank : 53 (local_rank: 5) - exitcode : 1 (pid: 89529) - args = _parse_args(extra_args_provider=extra_args_provider, - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - error_file: /tmp/torchelastic_qj1as6wk/none_82do5wit/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam35-ib0 - rank : 164 (local_rank: 4) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam15-ib0 - rank : 93 (local_rank: 5) - exitcode : 1 (pid: 92451) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam52-ib0 - rank : 287 (local_rank: 7) - exitcode : 1 (pid: 81478) - error_file: /tmp/torchelastic_vw2jgzlb/none_3_1r0v7g/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - exitcode : 1 (pid: 164333) - error_file: /tmp/torchelastic_qxt35ay2/none_a9dp3k0h/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam20-ib0 - rank : 119 (local_rank: 7) - exitcode : 1 (pid: 86544) - error_file: /tmp/torchelastic_cdzsak1n/none_k7ljn2kh/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam04-ib0 - rank : 14 (local_rank: 6) - exitcode : 1 (pid: 94498) - error_file: /tmp/torchelastic__z99v7jd/none_wzje8khg/attempt_0/6/error.json - traceback : Traceback (most recent call last): - error_file: /tmp/torchelastic_xajhzjf7/none_xnzgml15/attempt_0/5/error.json - traceback : Traceback (most recent call last): - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam09-ib0 - rank : 54 (local_rank: 6) - exitcode : 1 (pid: 89530) - error_file: /tmp/torchelastic_qj1as6wk/none_82do5wit/attempt_0/6/error.json - traceback : Traceback (most recent call last): - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam35-ib0 - rank : 165 (local_rank: 5) - exitcode : 1 (pid: 164334) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - error_file: /tmp/torchelastic_qxt35ay2/none_a9dp3k0h/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam15-ib0 - rank : 94 (local_rank: 6) - exitcode : 1 (pid: 92452) - error_file: /tmp/torchelastic_xajhzjf7/none_xnzgml15/attempt_0/6/error.json - traceback : Traceback (most recent call last): - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam52-ib0 - rank : 280 (local_rank: 0) - exitcode : 1 (pid: 81471) - error_file: /tmp/torchelastic_vw2jgzlb/none_3_1r0v7g/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam20-ib0 - rank : 112 (local_rank: 0) - exitcode : 1 (pid: 86537) - error_file: /tmp/torchelastic_cdzsak1n/none_k7ljn2kh/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam04-ib0 - rank : 15 (local_rank: 7) - exitcode : 1 (pid: 94499) - error_file: /tmp/torchelastic__z99v7jd/none_wzje8khg/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - run(args) - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam09-ib0 - rank : 55 (local_rank: 7) - exitcode : 1 (pid: 89531) - error_file: /tmp/torchelastic_qj1as6wk/none_82do5wit/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam35-ib0 - rank : 166 (local_rank: 6) - exitcode : 1 (pid: 164335) - error_file: /tmp/torchelastic_qxt35ay2/none_a9dp3k0h/attempt_0/6/error.json - traceback : Traceback (most recent call last): - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam15-ib0 - rank : 95 (local_rank: 7) - exitcode : 1 (pid: 92453) - error_file: /tmp/torchelastic_xajhzjf7/none_xnzgml15/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam04-ib0 - rank : 8 (local_rank: 0) - exitcode : 1 (pid: 94492) - error_file: /tmp/torchelastic__z99v7jd/none_wzje8khg/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam09-ib0 - rank : 48 (local_rank: 0) - exitcode : 1 (pid: 89524) - error_file: /tmp/torchelastic_qj1as6wk/none_82do5wit/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam35-ib0 - rank : 167 (local_rank: 7) - exitcode : 1 (pid: 164336) - error_file: /tmp/torchelastic_qxt35ay2/none_a9dp3k0h/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam15-ib0 - rank : 88 (local_rank: 0) - exitcode : 1 (pid: 92446) - error_file: /tmp/torchelastic_xajhzjf7/none_xnzgml15/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - run(args) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam35-ib0 - rank : 160 (local_rank: 0) - exitcode : 1 (pid: 164329) - error_file: /tmp/torchelastic_qxt35ay2/none_a9dp3k0h/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - raise ChildFailedError( - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam13-ib0 - rank : 73 (local_rank: 1) - exitcode : 1 (pid: 86977) - error_file: /tmp/torchelastic_fz3i8qoc/none_yo5frjyh/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam13-ib0 - rank : 74 (local_rank: 2) - exitcode : 1 (pid: 86978) - error_file: /tmp/torchelastic_fz3i8qoc/none_yo5frjyh/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - raise ChildFailedError( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam13-ib0 - rank : 75 (local_rank: 3) - exitcode : 1 (pid: 86979) - error_file: /tmp/torchelastic_fz3i8qoc/none_yo5frjyh/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam13-ib0 - rank : 76 (local_rank: 4) - exitcode : 1 (pid: 86980) - error_file: /tmp/torchelastic_fz3i8qoc/none_yo5frjyh/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam13-ib0 - rank : 77 (local_rank: 5) - exitcode : 1 (pid: 86981) - elastic_launch( - error_file: /tmp/torchelastic_fz3i8qoc/none_yo5frjyh/attempt_0/5/error.json - traceback : Traceback (most recent call last): -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam06-ib0 - rank : 25 (local_rank: 1) - exitcode : 1 (pid: 106804) - error_file: /tmp/torchelastic_v1cq2mtq/none_fu5874sl/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam13-ib0 - rank : 78 (local_rank: 6) - exitcode : 1 (pid: 86982) - error_file: /tmp/torchelastic_fz3i8qoc/none_yo5frjyh/attempt_0/6/error.json - traceback : Traceback (most recent call last): - raise ChildFailedError( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam13-ib0 - rank : 79 (local_rank: 7) - exitcode : 1 (pid: 86983) - error_file: /tmp/torchelastic_fz3i8qoc/none_yo5frjyh/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam06-ib0 - rank : 26 (local_rank: 2) - exitcode : 1 (pid: 106805) - error_file: /tmp/torchelastic_v1cq2mtq/none_fu5874sl/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam13-ib0 - rank : 72 (local_rank: 0) - exitcode : 1 (pid: 86976) - error_file: /tmp/torchelastic_fz3i8qoc/none_yo5frjyh/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam47-ib0 - rank : 257 (local_rank: 1) - exitcode : 1 (pid: 101450) - error_file: /tmp/torchelastic_dcq62wcs/none_lrpz6obh/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - main() - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam06-ib0 - rank : 27 (local_rank: 3) - exitcode : 1 (pid: 106806) - error_file: /tmp/torchelastic_v1cq2mtq/none_fu5874sl/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - main() - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam06-ib0 - rank : 28 (local_rank: 4) - exitcode : 1 (pid: 106807) - error_file: /tmp/torchelastic_v1cq2mtq/none_fu5874sl/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam06-ib0 - rank : 29 (local_rank: 5) - exitcode : 1 (pid: 106808) - error_file: /tmp/torchelastic_v1cq2mtq/none_fu5874sl/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - raise ChildFailedError( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam06-ib0 - rank : 30 (local_rank: 6) - exitcode : 1 (pid: 106809) - error_file: /tmp/torchelastic_v1cq2mtq/none_fu5874sl/attempt_0/6/error.json - traceback : Traceback (most recent call last): - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam47-ib0 - rank : 258 (local_rank: 2) - exitcode : 1 (pid: 101451) - error_file: /tmp/torchelastic_dcq62wcs/none_lrpz6obh/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam06-ib0 - rank : 31 (local_rank: 7) - exitcode : 1 (pid: 106810) - error_file: /tmp/torchelastic_v1cq2mtq/none_fu5874sl/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - return f(*args, **kwargs) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam28-ib0 - rank : 145 (local_rank: 1) - exitcode : 1 (pid: 97294) - error_file: /tmp/torchelastic_6_9fp90n/none_q678t89y/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam06-ib0 - rank : 24 (local_rank: 0) - exitcode : 1 (pid: 106803) - error_file: /tmp/torchelastic_v1cq2mtq/none_fu5874sl/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam47-ib0 - rank : 259 (local_rank: 3) - exitcode : 1 (pid: 101452) - error_file: /tmp/torchelastic_dcq62wcs/none_lrpz6obh/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam47-ib0 - rank : 260 (local_rank: 4) - exitcode : 1 (pid: 101453) - error_file: /tmp/torchelastic_dcq62wcs/none_lrpz6obh/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam47-ib0 - rank : 261 (local_rank: 5) - exitcode : 1 (pid: 101454) - error_file: /tmp/torchelastic_dcq62wcs/none_lrpz6obh/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam47-ib0 - rank : 262 (local_rank: 6) - exitcode : 1 (pid: 101455) - error_file: /tmp/torchelastic_dcq62wcs/none_lrpz6obh/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam28-ib0 - rank : 146 (local_rank: 2) - exitcode : 1 (pid: 97295) - error_file: /tmp/torchelastic_6_9fp90n/none_q678t89y/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam47-ib0 - rank : 263 (local_rank: 7) - exitcode : 1 (pid: 101456) - error_file: /tmp/torchelastic_dcq62wcs/none_lrpz6obh/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam28-ib0 - rank : 148 (local_rank: 4) - exitcode : 1 (pid: 97297) - error_file: /tmp/torchelastic_6_9fp90n/none_q678t89y/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam47-ib0 - rank : 256 (local_rank: 0) - exitcode : 1 (pid: 101449) - error_file: /tmp/torchelastic_dcq62wcs/none_lrpz6obh/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - run(args) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam28-ib0 - rank : 150 (local_rank: 6) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - exitcode : 1 (pid: 97299) - error_file: /tmp/torchelastic_6_9fp90n/none_q678t89y/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam28-ib0 - rank : 151 (local_rank: 7) - exitcode : 1 (pid: 97300) - error_file: /tmp/torchelastic_6_9fp90n/none_q678t89y/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam28-ib0 - rank : 144 (local_rank: 0) - exitcode : 1 (pid: 97293) - error_file: /tmp/torchelastic_6_9fp90n/none_q678t89y/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - elastic_launch( - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - return launch_agent(self._config, self._entrypoint, list(args)) - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - raise ChildFailedError( - raise ChildFailedError( - raise ChildFailedError( - raise ChildFailedError( - return launch_agent(self._config, self._entrypoint, list(args)) - raise ChildFailedError( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam40-ib0 - rank : 202 (local_rank: 2) - exitcode : 1 (pid: 614191) - error_file: /tmp/torchelastic_xgl4gquo/none_eg10gm5f/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam14-ib0 - rank : 81 (local_rank: 1) - exitcode : 1 (pid: 113790) - error_file: /tmp/torchelastic_h93zux93/none_hazade_1/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam37-ib0 - rank : 177 (local_rank: 1) - exitcode : 1 (pid: 613526) - error_file: /tmp/torchelastic_q8htwnkf/none_kjtzutn0/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam24-ib0 - rank : 129 (local_rank: 1) - exitcode : 1 (pid: 93650) - error_file: /tmp/torchelastic_bqw88oz7/none_smdcysq5/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam30-ib0 - rank : 153 (local_rank: 1) - exitcode : 1 (pid: 94494) - error_file: /tmp/torchelastic_fpc_p_qf/none_3e19hdpy/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - raise ChildFailedError( - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam40-ib0 - rank : 203 (local_rank: 3) - exitcode : 1 (pid: 614192) - error_file: /tmp/torchelastic_xgl4gquo/none_eg10gm5f/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam14-ib0 - rank : 82 (local_rank: 2) - exitcode : 1 (pid: 113791) - error_file: /tmp/torchelastic_h93zux93/none_hazade_1/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam37-ib0 - rank : 178 (local_rank: 2) - exitcode : 1 (pid: 613527) - error_file: /tmp/torchelastic_q8htwnkf/none_kjtzutn0/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam30-ib0 - rank : 154 (local_rank: 2) - exitcode : 1 (pid: 94495) - error_file: /tmp/torchelastic_fpc_p_qf/none_3e19hdpy/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:05 - host : jean-zay-iam24-ib0 - rank : 130 (local_rank: 2) - exitcode : 1 (pid: 93651) - error_file: /tmp/torchelastic_bqw88oz7/none_smdcysq5/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam11-ib0 - rank : 65 (local_rank: 1) - exitcode : 1 (pid: 105531) - error_file: /tmp/torchelastic_ezwg5ryv/none_x88nqw90/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam40-ib0 - rank : 204 (local_rank: 4) - exitcode : 1 (pid: 614193) - error_file: /tmp/torchelastic_xgl4gquo/none_eg10gm5f/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam14-ib0 - rank : 84 (local_rank: 4) - exitcode : 1 (pid: 113793) - error_file: /tmp/torchelastic_h93zux93/none_hazade_1/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam37-ib0 - rank : 179 (local_rank: 3) - exitcode : 1 (pid: 613528) - error_file: /tmp/torchelastic_q8htwnkf/none_kjtzutn0/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam24-ib0 - rank : 131 (local_rank: 3) - exitcode : 1 (pid: 93652) - error_file: /tmp/torchelastic_bqw88oz7/none_smdcysq5/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam30-ib0 - rank : 155 (local_rank: 3) - exitcode : 1 (pid: 94496) - error_file: /tmp/torchelastic_fpc_p_qf/none_3e19hdpy/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam40-ib0 - rank : 205 (local_rank: 5) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam14-ib0 - rank : 85 (local_rank: 5) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - exitcode : 1 (pid: 614194) - error_file: /tmp/torchelastic_xgl4gquo/none_eg10gm5f/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - exitcode : 1 (pid: 113794) - error_file: /tmp/torchelastic_h93zux93/none_hazade_1/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam37-ib0 - rank : 180 (local_rank: 4) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam24-ib0 - rank : 132 (local_rank: 4) - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam11-ib0 - rank : 66 (local_rank: 2) - exitcode : 1 (pid: 105532) - error_file: /tmp/torchelastic_ezwg5ryv/none_x88nqw90/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam30-ib0 - rank : 156 (local_rank: 4) - exitcode : 1 (pid: 613529) - error_file: /tmp/torchelastic_q8htwnkf/none_kjtzutn0/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - exitcode : 1 (pid: 93653) - error_file: /tmp/torchelastic_bqw88oz7/none_smdcysq5/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam40-ib0 - rank : 206 (local_rank: 6) - exitcode : 1 (pid: 614195) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam14-ib0 - rank : 86 (local_rank: 6) - exitcode : 1 (pid: 113795) - exitcode : 1 (pid: 94497) - error_file: /tmp/torchelastic_fpc_p_qf/none_3e19hdpy/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - error_file: /tmp/torchelastic_xgl4gquo/none_eg10gm5f/attempt_0/6/error.json - traceback : Traceback (most recent call last): - error_file: /tmp/torchelastic_h93zux93/none_hazade_1/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam37-ib0 - rank : 181 (local_rank: 5) - exitcode : 1 (pid: 613530) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:05 - host : jean-zay-iam24-ib0 - rank : 133 (local_rank: 5) - exitcode : 1 (pid: 93654) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam30-ib0 - rank : 157 (local_rank: 5) - exitcode : 1 (pid: 94498) - error_file: /tmp/torchelastic_q8htwnkf/none_kjtzutn0/attempt_0/5/error.json - traceback : Traceback (most recent call last): - error_file: /tmp/torchelastic_bqw88oz7/none_smdcysq5/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - error_file: /tmp/torchelastic_fpc_p_qf/none_3e19hdpy/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam40-ib0 - rank : 201 (local_rank: 1) - exitcode : 1 (pid: 614190) - error_file: /tmp/torchelastic_xgl4gquo/none_eg10gm5f/attempt_0/1/error.json - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam14-ib0 - rank : 80 (local_rank: 0) - exitcode : 1 (pid: 113789) - error_file: /tmp/torchelastic_h93zux93/none_hazade_1/attempt_0/0/error.json - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam11-ib0 - rank : 67 (local_rank: 3) - exitcode : 1 (pid: 105533) - error_file: /tmp/torchelastic_ezwg5ryv/none_x88nqw90/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam37-ib0 - rank : 182 (local_rank: 6) - exitcode : 1 (pid: 613531) - error_file: /tmp/torchelastic_q8htwnkf/none_kjtzutn0/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam24-ib0 - rank : 134 (local_rank: 6) - exitcode : 1 (pid: 93655) - error_file: /tmp/torchelastic_bqw88oz7/none_smdcysq5/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam30-ib0 - rank : 158 (local_rank: 6) - exitcode : 1 (pid: 94499) - error_file: /tmp/torchelastic_fpc_p_qf/none_3e19hdpy/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam11-ib0 - rank : 68 (local_rank: 4) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - exitcode : 1 (pid: 105534) - error_file: /tmp/torchelastic_ezwg5ryv/none_x88nqw90/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam37-ib0 - rank : 183 (local_rank: 7) - exitcode : 1 (pid: 613532) - error_file: /tmp/torchelastic_q8htwnkf/none_kjtzutn0/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam24-ib0 - rank : 135 (local_rank: 7) - exitcode : 1 (pid: 93656) - error_file: /tmp/torchelastic_bqw88oz7/none_smdcysq5/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam30-ib0 - rank : 159 (local_rank: 7) - exitcode : 1 (pid: 94500) - error_file: /tmp/torchelastic_fpc_p_qf/none_3e19hdpy/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam11-ib0 - rank : 69 (local_rank: 5) - exitcode : 1 (pid: 105535) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - error_file: /tmp/torchelastic_ezwg5ryv/none_x88nqw90/attempt_0/5/error.json - traceback : Traceback (most recent call last): - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam37-ib0 - rank : 176 (local_rank: 0) - exitcode : 1 (pid: 613525) - error_file: /tmp/torchelastic_q8htwnkf/none_kjtzutn0/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam24-ib0 - rank : 128 (local_rank: 0) - exitcode : 1 (pid: 93649) - error_file: /tmp/torchelastic_bqw88oz7/none_smdcysq5/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam30-ib0 - rank : 152 (local_rank: 0) - exitcode : 1 (pid: 94493) - error_file: /tmp/torchelastic_fpc_p_qf/none_3e19hdpy/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam11-ib0 - rank : 70 (local_rank: 6) - exitcode : 1 (pid: 105536) - error_file: /tmp/torchelastic_ezwg5ryv/none_x88nqw90/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam11-ib0 - rank : 71 (local_rank: 7) - exitcode : 1 (pid: 105537) - error_file: /tmp/torchelastic_ezwg5ryv/none_x88nqw90/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam11-ib0 - rank : 64 (local_rank: 0) - exitcode : 1 (pid: 105530) - error_file: /tmp/torchelastic_ezwg5ryv/none_x88nqw90/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam41-ib0 - rank : 209 (local_rank: 1) - exitcode : 1 (pid: 123545) - error_file: /tmp/torchelastic_l31eaw53/none_gto2do6d/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - raise ChildFailedError( - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam41-ib0 - rank : 210 (local_rank: 2) - exitcode : 1 (pid: 123546) - error_file: /tmp/torchelastic_l31eaw53/none_gto2do6d/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam41-ib0 - rank : 211 (local_rank: 3) - exitcode : 1 (pid: 123547) - error_file: /tmp/torchelastic_l31eaw53/none_gto2do6d/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py FAILED ------------------------------------------------------------- -Failures: -[1]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam07-ib0 - rank : 33 (local_rank: 1) - exitcode : 1 (pid: 106877) - error_file: /tmp/torchelastic_4cb0ut8m/none_2cpc2dj6/attempt_0/1/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam41-ib0 - rank : 212 (local_rank: 4) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - exitcode : 1 (pid: 123548) - error_file: /tmp/torchelastic_l31eaw53/none_gto2do6d/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam41-ib0 - rank : 213 (local_rank: 5) - exitcode : 1 (pid: 123549) - error_file: /tmp/torchelastic_l31eaw53/none_gto2do6d/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam41-ib0 - rank : 214 (local_rank: 6) - exitcode : 1 (pid: 123550) - error_file: /tmp/torchelastic_l31eaw53/none_gto2do6d/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[2]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam07-ib0 - rank : 34 (local_rank: 2) - exitcode : 1 (pid: 106878) - error_file: /tmp/torchelastic_4cb0ut8m/none_2cpc2dj6/attempt_0/2/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam41-ib0 - rank : 215 (local_rank: 7) - exitcode : 1 (pid: 123551) - error_file: /tmp/torchelastic_l31eaw53/none_gto2do6d/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam41-ib0 - rank : 208 (local_rank: 0) - exitcode : 1 (pid: 123544) - error_file: /tmp/torchelastic_l31eaw53/none_gto2do6d/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[3]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam07-ib0 - rank : 35 (local_rank: 3) - exitcode : 1 (pid: 106879) - error_file: /tmp/torchelastic_4cb0ut8m/none_2cpc2dj6/attempt_0/3/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[4]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam07-ib0 - rank : 36 (local_rank: 4) - exitcode : 1 (pid: 106880) - error_file: /tmp/torchelastic_4cb0ut8m/none_2cpc2dj6/attempt_0/4/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[5]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam07-ib0 - rank : 37 (local_rank: 5) - exitcode : 1 (pid: 106881) - error_file: /tmp/torchelastic_4cb0ut8m/none_2cpc2dj6/attempt_0/5/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[6]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam07-ib0 - rank : 38 (local_rank: 6) - exitcode : 1 (pid: 106882) - error_file: /tmp/torchelastic_4cb0ut8m/none_2cpc2dj6/attempt_0/6/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -[7]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam07-ib0 - rank : 39 (local_rank: 7) - exitcode : 1 (pid: 106883) - error_file: /tmp/torchelastic_4cb0ut8m/none_2cpc2dj6/attempt_0/7/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:04 - host : jean-zay-iam07-ib0 - rank : 32 (local_rank: 0) - exitcode : 1 (pid: 106876) - error_file: /tmp/torchelastic_4cb0ut8m/none_2cpc2dj6/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2022-09-07_21:51:03 - host : jean-zay-iam44-ib0 - rank : 232 (local_rank: 0) - exitcode : 1 (pid: 613483) - error_file: /tmp/torchelastic_6ewpocrd/none_riyse4wl/attempt_0/0/error.json - traceback : Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/finetune_t0.py", line 199, in main - pretrain( - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/training.py", line 99, in pretrain - initialize_megatron(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/initialize.py", line 87, in initialize_megatron - set_global_variables(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 90, in set_global_variables - args = _parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/global_vars.py", line 107, in _parse_args - _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 67, in parse_args - args = parser.parse_args() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1768, in parse_args - args, argv = self.parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1800, in parse_known_args - namespace, args = self._parse_known_args(args, namespace) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 2006, in _parse_known_args - start_index = consume_optional(start_index) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1946, in consume_optional - take_action(action, args, option_string) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/argparse.py", line 1874, in take_action - action(self, namespace, argument_values, option_string) - File "/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/arguments.py", line 865, in __call__ - with open(values, "r") as fi: - FileNotFoundError: [Errno 2] No such file or directory: '/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/Megatron-DeepSpeed/data/xp3zz_train.txt' - -============================================================ -srun: error: jean-zay-iam51: task 34: Exited with exit code 1 -srun: launch/slurm: _step_signal: Terminating StepId=1096011.0 -srun: error: jean-zay-iam50: task 33: Exited with exit code 1 -slurmstepd: error: *** STEP 1096011.0 ON jean-zay-iam03 CANCELLED AT 2022-09-07T21:51:11 *** -srun: error: jean-zay-iam19: task 13: Exited with exit code 1 -srun: error: jean-zay-iam21: task 15: Exited with exit code 1 -srun: error: jean-zay-iam18: task 12: Exited with exit code 1 -srun: error: jean-zay-iam04: task 1: Exited with exit code 1 -srun: error: jean-zay-iam52: task 35: Exited with exit code 1 -srun: error: jean-zay-iam20: task 14: Exited with exit code 1 -srun: error: jean-zay-iam05: task 2: Exited with exit code 1 -srun: error: jean-zay-iam07: task 4: Exited with exit code 1 -srun: error: jean-zay-iam24: task 16: Exited with exit code 1 -srun: error: jean-zay-iam10: task 7: Exited with exit code 1 -srun: error: jean-zay-iam25: task 17: Exited with exit code 1 -srun: error: jean-zay-iam13: task 9: Exited with exit code 1 -srun: error: jean-zay-iam15: task 11: Exited with exit code 1 -srun: error: jean-zay-iam40: task 25: Exited with exit code 1 -srun: error: jean-zay-iam09: task 6: Exited with exit code 1 -srun: error: jean-zay-iam03: task 0: Exited with exit code 1 -srun: error: jean-zay-iam39: task 24: Exited with exit code 1 -srun: error: jean-zay-iam47: task 32: Exited with exit code 1 -srun: error: jean-zay-iam14: task 10: Exited with exit code 1 -srun: error: jean-zay-iam28: task 18: Exited with exit code 1 -srun: error: jean-zay-iam36: task 21: Exited with exit code 1 -srun: error: jean-zay-iam42: task 27: Exited with exit code 1 -srun: error: jean-zay-iam41: task 26: Exited with exit code 1 -srun: error: jean-zay-iam45: task 30: Exited with exit code 1 -srun: error: jean-zay-iam44: task 29: Exited with exit code 1 -srun: error: jean-zay-iam38: task 23: Exited with exit code 1 -srun: error: jean-zay-iam37: task 22: Exited with exit code 1 -srun: error: jean-zay-iam08: task 5: Exited with exit code 1 -srun: error: jean-zay-iam35: task 20: Exited with exit code 1 -srun: error: jean-zay-iam30: task 19: Exited with exit code 1 -srun: error: jean-zay-iam11: task 8: Exited with exit code 1 -srun: error: jean-zay-iam46: task 31: Exited with exit code 1 -srun: error: jean-zay-iam43: task 28: Exited with exit code 1 -srun: error: jean-zay-iam06: task 3: Exited with exit code 1 -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[default0]:using world size: 288, data-parallel-size: 4, tensor-model-parallel size: 1, pipeline-model-parallel size: 72 -[default0]:accumulate and all-reduce gradients in fp32 for bfloat16 data type. -[default0]:using torch.bfloat16 for parameters ... -[default0]:------------------------ arguments ------------------------ -[default0]: abort_on_unmet_fused_kernel_constraints ......... True -[default0]: accumulate_allreduce_grads_in_fp32 .............. True -[default0]: adam_beta1 ...................................... 0.9 -[default0]: adam_beta2 ...................................... 0.95 -[default0]: adam_eps ........................................ 1e-08 -[default0]: adlr_autoresume ................................. False -[default0]: adlr_autoresume_interval ........................ 1000 -[default0]: apply_query_key_layer_scaling ................... True -[default0]: apply_residual_connection_post_layernorm ........ False -[default0]: attention_dropout ............................... 0.1 -[default0]: attention_softmax_in_fp32 ....................... False -[default0]: bert_binary_head ................................ True -[default0]: bert_load ....................................... None -[default0]: bf16 ............................................ True -[default0]: bias_dropout_fusion ............................. True -[default0]: bias_gelu_fusion ................................ True -[default0]: biencoder_projection_dim ........................ 0 -[default0]: biencoder_shared_query_context_model ............ False -[default0]: block_data_path ................................. None -[default0]: checkpoint_activations .......................... True -[default0]: checkpoint_in_cpu ............................... False -[default0]: checkpoint_num_layers ........................... 1 -[default0]: clip_grad ....................................... 1.0 -[default0]: codecarbon_dir .................................. None -[default0]: consumed_train_samples .......................... 0 -[default0]: consumed_train_tokens ........................... 0 -[default0]: consumed_valid_samples .......................... 0 -[default0]: contigious_checkpointing ........................ False -[default0]: cpu_optimizer ................................... False -[default0]: cpu_torch_adam .................................. False -[default0]: curriculum_learning ............................. False -[default0]: data_impl ....................................... mmap -[default0]: data_parallel_size .............................. 4 -[default0]: data_path ....................................... None -[default0]: dataloader_type ................................. single -[default0]: DDP_impl ........................................ local -[default0]: decoder_seq_length .............................. None -[default0]: deepscale ....................................... False -[default0]: deepscale_config ................................ None -[default0]: deepspeed ....................................... True -[default0]: deepspeed_activation_checkpointing .............. True -[default0]: deepspeed_config ................................ ./ds_config.1096018.json -[default0]: deepspeed_mpi ................................... False -[default0]: distribute_checkpointed_activations ............. False -[default0]: distributed_backend ............................. nccl -[default0]: embed_layernorm ................................. True -[default0]: embedding_path .................................. None -[default0]: encoder_seq_length .............................. 2048 -[default0]: eod_mask_loss ................................... False -[default0]: eval_interval ................................... 250 -[default0]: eval_iters ...................................... 1 -[default0]: eval_only ....................................... None -[default0]: evidence_data_path .............................. None -[default0]: exit_duration_in_mins ........................... 5990 -[default0]: exit_interval ................................... None -[default0]: ffn_hidden_size ................................. 57344 -[default0]: finetune ........................................ False -[default0]: fp16 ............................................ False -[default0]: fp16_lm_cross_entropy ........................... False -[default0]: fp32_residual_connection ........................ False -[default0]: gigaflos_no_embeds .............................. 0 -[default0]: global_batch_size ............................... 2048 -[default0]: glu_activation .................................. None -[default0]: hidden_dropout .................................. 0.1 -[default0]: hidden_size ..................................... 14336 -[default0]: hysteresis ...................................... 2 -[default0]: ict_head_size ................................... None -[default0]: ict_load ........................................ None -[default0]: img_dim ......................................... 224 -[default0]: indexer_batch_size .............................. 128 -[default0]: indexer_log_interval ............................ 1000 -[default0]: inference ....................................... False -[default0]: init_method_std ................................. 0.0048 -[default0]: init_method_xavier_uniform ...................... False -[default0]: initial_loss_scale .............................. 4294967296 -[default0]: kill_switch_path ................................ /gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/kill-switch-tr13-176B-mtf -[default0]: kv_channels ..................................... 128 -[default0]: layernorm_epsilon ............................... 1e-05 -[default0]: lazy_mpu_init ................................... None -[default0]: load ............................................ /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]: local_rank ...................................... None -[default0]: log_batch_size_to_tensorboard ................... True -[default0]: log_interval .................................... 1 -[default0]: log_learning_rate_to_tensorboard ................ True -[default0]: log_level ....................................... None -[default0]: log_level_replica ............................... None -[default0]: log_loss_scale_to_tensorboard ................... True -[default0]: log_num_zeros_in_grad ........................... False -[default0]: log_params_norm ................................. False -[default0]: log_path ........................................ None -[default0]: log_timers_to_tensorboard ....................... True -[default0]: log_validation_ppl_to_tensorboard ............... True -[default0]: loss_on_targets_only ............................ False -[default0]: loss_scale ...................................... None -[default0]: loss_scale_window ............................... 1000 -[default0]: lr .............................................. 2e-05 -[default0]: lr_decay_iters .................................. None -[default0]: lr_decay_samples ................................ None -[default0]: lr_decay_style .................................. constant -[default0]: lr_decay_tokens ................................. None -[default0]: lr_warmup_fraction .............................. None -[default0]: lr_warmup_iters ................................. 0 -[default0]: lr_warmup_samples ............................... 0 -[default0]: make_vocab_size_divisible_by .................... 128 -[default0]: mask_prob ....................................... 0.15 -[default0]: masked_softmax_fusion ........................... True -[default0]: max_position_embeddings ......................... 2048 -[default0]: mean_noise_span_length .......................... None -[default0]: memory_centric_tiled_linear ..................... False -[default0]: merge_file ...................................... None -[default0]: micro_batch_size ................................ 1 -[default0]: min_loss_scale .................................. 1.0 -[default0]: min_lr .......................................... 0.0 -[default0]: mmap_warmup ..................................... False -[default0]: no_load_optim ................................... True -[default0]: no_load_rng ..................................... None -[default0]: no_save_optim ................................... None -[default0]: no_save_rng ..................................... None -[default0]: noise_density ................................... None -[default0]: norm_target_loss ................................ True -[default0]: num_attention_heads ............................. 112 -[default0]: num_channels .................................... 3 -[default0]: num_classes ..................................... 1000 -[default0]: num_layers ...................................... 70 -[default0]: num_layers_per_virtual_pipeline_stage ........... None -[default0]: num_workers ..................................... 2 -[default0]: onnx_safe ....................................... None -[default0]: openai_gelu ..................................... False -[default0]: optimizer ....................................... adam -[default0]: override_lr_scheduler ........................... False -[default0]: pad_vocab_size_to ............................... 250880 -[default0]: params_dtype .................................... torch.bfloat16 -[default0]: partition_activations ........................... False -[default0]: patch_dim ....................................... 16 -[default0]: pipeline_model_parallel_size .................... 72 -[default0]: position_embedding_type ......................... PositionEmbeddingType.alibi -[default0]: pp_partition_method ............................. type:transformer|embedding -[default0]: prefixlm ........................................ False -[default0]: profile_backward ................................ False -[default0]: query_in_block_prob ............................. 0.1 -[default0]: rampup_batch_size ............................... None -[default0]: rank ............................................ 0 -[default0]: remote_device ................................... none -[default0]: reset_attention_mask ............................ False -[default0]: reset_position_ids .............................. False -[default0]: reset_progress .................................. True -[default0]: retriever_report_topk_accuracies ................ [] -[default0]: retriever_score_scaling ......................... False -[default0]: retriever_seq_length ............................ 256 -[default0]: reweight_loss_based_on_position_frequency ....... False -[default0]: sample_rate ..................................... 1.0 -[default0]: save ............................................ /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]: save_interval ................................... 5 -[default0]: scatter_gather_tensors_in_pipeline .............. True -[default0]: scattered_embeddings ............................ False -[default0]: seed ............................................ 42 -[default0]: seq_length ...................................... 2048 -[default0]: sgd_momentum .................................... 0.9 -[default0]: short_seq_prob .................................. 0.1 -[default0]: skip_train_iteration_range ...................... None -[default0]: split ........................................... None -[default0]: split_transformers .............................. False -[default0]: sync_tp_duplicated_parameters ................... True -[default0]: synchronize_each_layer .......................... False -[default0]: tensor_model_parallel_size ...................... 1 -[default0]: tensorboard_dir ................................. /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/tr13-176B-ml-t0-logs/tensorboard/xp3zzlossseq -[default0]: tensorboard_log_interval ........................ 1 -[default0]: tensorboard_queue_size .......................... 5 -[default0]: test_weighted_split_paths ....................... None -[default0]: test_weighted_split_paths_path .................. None -[default0]: tile_factor ..................................... 1 -[default0]: titles_data_path ................................ None -[default0]: tokenizer_name_or_path .......................... bigscience/tokenizer -[default0]: tokenizer_type .................................. PretrainedFromHF -[default0]: train_iters ..................................... None -[default0]: train_samples ................................... 6348800 -[default0]: train_tokens .................................... None -[default0]: train_weighted_split_names ...................... ['train'] -[default0]: train_weighted_split_paths ...................... [['/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_en', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_es', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pt', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fr', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_code', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ar', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_id', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zh', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_hi', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_vi', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ur', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_te', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ta', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bn', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_mr', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sw', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_gu', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pa', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ne', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_yo', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ig', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ny', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zu', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_xh', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sn', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ts', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rw', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_lg', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tn', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_nso', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rn', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ml', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_kn', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_or', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_as', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ln', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_wo', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tum', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ki', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_st', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fon', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_eu', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ca', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ak', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bm', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tw']] -[default0]: train_weighted_split_paths_path ................. None -[default0]: train_weighted_split_splits ..................... [['0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1']] -[default0]: train_weighted_split_weights .................... [['0.3932835937', '0.0860451087', '0.0690010451', '0.0660385329', '0.0590120118', '0.046925039', '0.0462116635', '0.0460832559', '0.0441207519', '0.031085057', '0.0192197788', '0.0134582697', '0.0092870269', '0.0083432872', '0.006675271', '0.0056775071', '0.0056177118', '0.0052425885', '0.0039444054', '0.0035346554', '0.0032586031', '0.0027265372', '0.0026422146', '0.00255164', '0.0025298379', '0.0025073947', '0.0024984173', '0.002363918', '0.0023599103', '0.0023015578', '0.0019336484', '0.0017537816', '0.0016577564', '0.0016178395', '0.0015655787', '0.00126548', '0.0012279677', '0.0011625616', '0.0011526224', '0.0011430039', '0.0011329044', '0.0011322632', '0.0011082168', '0.0010830483', '0.0010726282', '0.0010649334']] -[default0]: universal_checkpoint ............................ True -[default0]: use_bnb_optimizer ............................... False -[default0]: use_checkpoint_lr_scheduler ..................... False -[default0]: use_contiguous_buffers_in_ddp ................... True -[default0]: use_cpu_initialization .......................... None -[default0]: use_one_sent_docs ............................... False -[default0]: use_pin_memory .................................. False -[default0]: valid_num_workers ............................... 2 -[default0]: valid_weighted_split_names ...................... ['validation_pretraining'] -[default0]: valid_weighted_split_paths ...................... [['/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ar/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ar_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ca/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ca_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/code/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_code_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/en/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_en_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/es/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_es_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/eu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_eu_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/fr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_fr_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/id/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_id_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-as/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-as_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-bn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-bn_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-gu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-gu_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-hi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-hi_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-kn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-kn_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ml/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ml_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-mr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-mr_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ne/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ne_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-or/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-or_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-pa/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-pa_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ta/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ta_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-te/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-te_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ur/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ur_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/nigercongo-all/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_nigercongo-all_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-en/meg_ds_bigscience_tokenizer_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-zh/meg_ds_bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/pt/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_pt_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/vi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_vi_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zhs/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zhs_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zht/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zht_text_document']] -[default0]: valid_weighted_split_paths_path ................. None -[default0]: valid_weighted_split_splits ..................... [['0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0']] -[default0]: valid_weighted_split_weights .................... [['0.0330676168743166', '0.011242051312222764', '0.13027200903379185', '0.22171164529099704', '0.10667815627928671', '0.0015595123898173287', '0.13054018439603915', '0.01091803753667153', '0.00011021422347108609', '0.005492381453597748', '0.0004021215011318779', '0.007470068593492175', '0.0006190467776576425', '0.0010335296343329384', '0.0005012010684646179', '0.0006672772956128299', '0.00035928138344705506', '0.0005084433130291778', '0.0021137328219915496', '0.0009129946225980253', '0.0012454301613725426', '0.00031588689199263235', '0.08137213783015229', '0.055293935695898196', '0.04954150576361177', '0.02461641286531197', '0.12091748245519074', '0.0005177025345001541']] -[default0]: virtual_pipeline_model_parallel_size ............ None -[default0]: vocab_extra_ids ................................. 0 -[default0]: vocab_file ...................................... None -[default0]: weight_decay .................................... 0.0001 -[default0]: world_size ...................................... 288 -[default0]: zero_allgather_bucket_size ...................... 0.0 -[default0]: zero_contigious_gradients ....................... False -[default0]: zero_reduce_bucket_size ......................... 0.0 -[default0]: zero_reduce_scatter ............................. False -[default0]: zero_stage ...................................... 0 -[default0]:-------------------- end of arguments --------------------- -[default0]:setting number of micro-batches to constant 512 -[default0]:> building PretrainedFromHF tokenizer ... -[default0]: vocab file is un-used. loading tokenizer from pre-trained model -[default0]:Offline mode: forcing local_files_only=True -[default0]:Offline mode: forcing local_files_only=True -[default0]:loading file https://huggingface.co/bigscience/tokenizer/resolve/main/tokenizer.json from cache at /gpfswork/rech/six/commun/models/29d0a41f4527257b8afe6d5495f492dac260318430f18239a42ca5f6dc4487fc.7b0fb8edc2986944ff9b7418149b52d8c4a1354a17d0360deb8974da70c6cc03 -[default0]:loading file https://huggingface.co/bigscience/tokenizer/resolve/main/added_tokens.json from cache at None -[default0]:loading file https://huggingface.co/bigscience/tokenizer/resolve/main/special_tokens_map.json from cache at /gpfswork/rech/six/commun/models/4f03e43bcc54e0721823e6a06b1d197905e2ea79aa7dcc1a0f0fcecc73ce3fb2.9d6cd81ef646692fb1c169a880161ea1cb95f49694f220aced9b704b457e51dd -[default0]:loading file https://huggingface.co/bigscience/tokenizer/resolve/main/tokenizer_config.json from cache at /gpfswork/rech/six/commun/models/9441c67b923ef7a65950a64e31c40f80ed181ba59502981a80f2cd0c438c6432.3c09887250243e50d8de9d10b2a778152434f62a22a95b5f89dbbe79a6eb496a -[default0]: > padded vocab (size: 250680) with 200 dummy tokens (new size: 250880) -[default0]:DeepSpeed general environment info: -[default0]:torch install path ............... ['/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch'] -[default0]:torch version .................... 1.12.0 -[default0]:torch cuda version ............... 11.3 -[default0]:torch hip version ................ None -[default0]:nvcc version ..................... 11.4 -[default0]:deepspeed install path ........... ['/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/deepspeed'] -[default0]:deepspeed info ................... 0.7.1+8b2a6371, 8b2a6371, master -[default0]:deepspeed wheel compiled w. ...... torch 1.12, cuda 11.3 -[default0]:**** Git info for Megatron: git_hash=6c1018f git_branch=mtf-multival **** -[default0]:> initializing torch distributed ... -[default0]:[2022-09-07 21:53:21,191] [INFO] [comm.py:628:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl -[default7]:> setting tensorboard ... -[default0]:> initializing tensor model parallel with size 1 -[default0]:> initializing pipeline model parallel with size 72 -[default0]:> setting random seeds to 42 ... -[default0]:[2022-09-07 21:53:30,312] [INFO] [checkpointing.py:226:model_parallel_cuda_manual_seed] > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 2760 and data parallel seed: 42 -[default0]:> compiling dataset index builder ... -[default0]:make: Entering directory '/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/data' -[default0]:make: Nothing to be done for 'default'. -[default0]:make: Leaving directory '/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/data' -[default0]:>>> done with dataset index builder. Compilation time: 0.087 seconds -[default0]:> compiling and loading fused kernels ... -[default0]:Detected CUDA files, patching ldflags -[default0]:Emitting ninja build file /gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... -[default0]:Building extension module scaled_upper_triang_masked_softmax_cuda... -[default0]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default0]:ninja: no work to do. -[default0]:Loading extension module scaled_upper_triang_masked_softmax_cuda... -[default0]:Detected CUDA files, patching ldflags -[default0]:Emitting ninja build file /gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... -[default0]:Building extension module scaled_masked_softmax_cuda... -[default0]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default0]:ninja: no work to do. -[default0]:Loading extension module scaled_masked_softmax_cuda... -[default0]:Detected CUDA files, patching ldflags -[default0]:Emitting ninja build file /gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... -[default0]:Building extension module fused_mix_prec_layer_norm_cuda... -[default0]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default0]:ninja: no work to do. -[default0]:Loading extension module fused_mix_prec_layer_norm_cuda... -[default0]:>>> done with compiling and loading fused kernels. Compilation time: 8.181 seconds -[default0]:time to initialize megatron (seconds): 66.582 -[default0]:[after megatron is initialized] datetime: 2022-09-07 21:53:38 -[default0]:building GPT model ... -[default0]:[2022-09-07 21:53:38,623] [INFO] [utils.py:827:see_memory_usage] Before Building Model -[default0]:[2022-09-07 21:53:38,624] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB -[default0]:[2022-09-07 21:53:38,624] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.85 GB, percent = 6.1% -[default0]:SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None -[default0]:Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=1, data=0, model=0): 4, ProcessCoord(pipe=1, data=1, model=0): 5, ProcessCoord(pipe=1, data=2, model=0): 6, ProcessCoord(pipe=1, data=3, model=0): 7, ProcessCoord(pipe=2, data=0, model=0): 8, ProcessCoord(pipe=2, data=1, model=0): 9, ProcessCoord(pipe=2, data=2, model=0): 10, ProcessCoord(pipe=2, data=3, model=0): 11, ProcessCoord(pipe=3, data=0, model=0): 12, ProcessCoord(pipe=3, data=1, model=0): 13, ProcessCoord(pipe=3, data=2, model=0): 14, ProcessCoord(pipe=3, data=3, model=0): 15, ProcessCoord(pipe=4, data=0, model=0): 16, ProcessCoord(pipe=4, data=1, model=0): 17, ProcessCoord(pipe=4, data=2, model=0): 18, ProcessCoord(pipe=4, data=3, model=0): 19, ProcessCoord(pipe=5, data=0, model=0): 20, ProcessCoord(pipe=5, data=1, model=0): 21, ProcessCoord(pipe=5, data=2, model=0): 22, ProcessCoord(pipe=5, data=3, model=0): 23, ProcessCoord(pipe=6, data=0, model=0): 24, ProcessCoord(pipe=6, data=1, model=0): 25, ProcessCoord(pipe=6, data=2, model=0): 26, ProcessCoord(pipe=6, data=3, model=0): 27, ProcessCoord(pipe=7, data=0, model=0): 28, ProcessCoord(pipe=7, data=1, model=0): 29, ProcessCoord(pipe=7, data=2, model=0): 30, ProcessCoord(pipe=7, data=3, model=0): 31, ProcessCoord(pipe=8, data=0, model=0): 32, ProcessCoord(pipe=8, data=1, model=0): 33, ProcessCoord(pipe=8, data=2, model=0): 34, ProcessCoord(pipe=8, data=3, model=0): 35, ProcessCoord(pipe=9, data=0, model=0): 36, ProcessCoord(pipe=9, data=1, model=0): 37, ProcessCoord(pipe=9, data=2, model=0): 38, ProcessCoord(pipe=9, data=3, model=0): 39, ProcessCoord(pipe=10, data=0, model=0): 40, ProcessCoord(pipe=10, data=1, model=0): 41, ProcessCoord(pipe=10, data=2, model=0): 42, ProcessCoord(pipe=10, data=3, model=0): 43, ProcessCoord(pipe=11, data=0, model=0): 44, ProcessCoord(pipe=11, data=1, model=0): 45, ProcessCoord(pipe=11, data=2, model=0): 46, ProcessCoord(pipe=11, data=3, model=0): 47, ProcessCoord(pipe=12, data=0, model=0): 48, ProcessCoord(pipe=12, data=1, model=0): 49, ProcessCoord(pipe=12, data=2, model=0): 50, ProcessCoord(pipe=12, data=3, model=0): 51, ProcessCoord(pipe=13, data=0, model=0): 52, ProcessCoord(pipe=13, data=1, model=0): 53, ProcessCoord(pipe=13, data=2, model=0): 54, ProcessCoord(pipe=13, data=3, model=0): 55, ProcessCoord(pipe=14, data=0, model=0): 56, ProcessCoord(pipe=14, data=1, model=0): 57, ProcessCoord(pipe=14, data=2, model=0): 58, ProcessCoord(pipe=14, data=3, model=0): 59, ProcessCoord(pipe=15, data=0, model=0): 60, ProcessCoord(pipe=15, data=1, model=0): 61, ProcessCoord(pipe=15, data=2, model=0): 62, ProcessCoord(pipe=15, data=3, model=0): 63, ProcessCoord(pipe=16, data=0, model=0): 64, ProcessCoord(pipe=16, data=1, model=0): 65, ProcessCoord(pipe=16, data=2, model=0): 66, ProcessCoord(pipe=16, data=3, model=0): 67, ProcessCoord(pipe=17, data=0, model=0): 68, ProcessCoord(pipe=17, data=1, model=0): 69, ProcessCoord(pipe=17, data=2, model=0): 70, ProcessCoord(pipe=17, data=3, model=0): 71, ProcessCoord(pipe=18, data=0, model=0): 72, ProcessCoord(pipe=18, data=1, model=0): 73, ProcessCoord(pipe=18, data=2, model=0): 74, ProcessCoord(pipe=18, data=3, model=0): 75, ProcessCoord(pipe=19, data=0, model=0): 76, ProcessCoord(pipe=19, data=1, model=0): 77, ProcessCoord(pipe=19, data=2, model=0): 78, ProcessCoord(pipe=19, data=3, model=0): 79, ProcessCoord(pipe=20, data=0, model=0): 80, ProcessCoord(pipe=20, data=1, model=0): 81, ProcessCoord(pipe=20, data=2, model=0): 82, ProcessCoord(pipe=20, data=3, model=0): 83, ProcessCoord(pipe=21, data=0, model=0): 84, ProcessCoord(pipe=21, data=1, model=0): 85, ProcessCoord(pipe=21, data=2, model=0): 86, ProcessCoord(pipe=21, data=3, model=0): 87, ProcessCoord(pipe=22, data=0, model=0): 88, ProcessCoord(pipe=22, data=1, model=0): 89, ProcessCoord(pipe=22, data=2, model=0): 90, ProcessCoord(pipe=22, data=3, model=0): 91, ProcessCoord(pipe=23, data=0, model=0): 92, ProcessCoord(pipe=23, data=1, model=0): 93, ProcessCoord(pipe=23, data=2, model=0): 94, ProcessCoord(pipe=23, data=3, model=0): 95, ProcessCoord(pipe=24, data=0, model=0): 96, ProcessCoord(pipe=24, data=1, model=0): 97, ProcessCoord(pipe=24, data=2, model=0): 98, ProcessCoord(pipe=24, data=3, model=0): 99, ProcessCoord(pipe=25, data=0, model=0): 100, ProcessCoord(pipe=25, data=1, model=0): 101, ProcessCoord(pipe=25, data=2, model=0): 102, ProcessCoord(pipe=25, data=3, model=0): 103, ProcessCoord(pipe=26, data=0, model=0): 104, ProcessCoord(pipe=26, data=1, model=0): 105, ProcessCoord(pipe=26, data=2, model=0): 106, ProcessCoord(pipe=26, data=3, model=0): 107, ProcessCoord(pipe=27, data=0, model=0): 108, ProcessCoord(pipe=27, data=1, model=0): 109, ProcessCoord(pipe=27, data=2, model=0): 110, ProcessCoord(pipe=27, data=3, model=0): 111, ProcessCoord(pipe=28, data=0, model=0): 112, ProcessCoord(pipe=28, data=1, model=0): 113, ProcessCoord(pipe=28, data=2, model=0): 114, ProcessCoord(pipe=28, data=3, model=0): 115, ProcessCoord(pipe=29, data=0, model=0): 116, ProcessCoord(pipe=29, data=1, model=0): 117, ProcessCoord(pipe=29, data=2, model=0): 118, ProcessCoord(pipe=29, data=3, model=0): 119, ProcessCoord(pipe=30, data=0, model=0): 120, ProcessCoord(pipe=30, data=1, model=0): 121, ProcessCoord(pipe=30, data=2, model=0): 122, ProcessCoord(pipe=30, data=3, model=0): 123, ProcessCoord(pipe=31, data=0, model=0): 124, ProcessCoord(pipe=31, data=1, model=0): 125, ProcessCoord(pipe=31, data=2, model=0): 126, ProcessCoord(pipe=31, data=3, model=0): 127, ProcessCoord(pipe=32, data=0, model=0): 128, ProcessCoord(pipe=32, data=1, model=0): 129, ProcessCoord(pipe=32, data=2, model=0): 130, ProcessCoord(pipe=32, data=3, model=0): 131, ProcessCoord(pipe=33, data=0, model=0): 132, ProcessCoord(pipe=33, data=1, model=0): 133, ProcessCoord(pipe=33, data=2, model=0): 134, ProcessCoord(pipe=33, data=3, model=0): 135, ProcessCoord(pipe=34, data=0, model=0): 136, ProcessCoord(pipe=34, data=1, model=0): 137, ProcessCoord(pipe=34, data=2, model=0): 138, ProcessCoord(pipe=34, data=3, model=0): 139, ProcessCoord(pipe=35, data=0, model=0): 140, ProcessCoord(pipe=35, data=1, model=0): 141, ProcessCoord(pipe=35, data=2, model=0): 142, ProcessCoord(pipe=35, data=3, model=0): 143, ProcessCoord(pipe=36, data=0, model=0): 144, ProcessCoord(pipe=36, data=1, model=0): 145, ProcessCoord(pipe=36, data=2, model=0): 146, ProcessCoord(pipe=36, data=3, model=0): 147, ProcessCoord(pipe=37, data=0, model=0): 148, ProcessCoord(pipe=37, data=1, model=0): 149, ProcessCoord(pipe=37, data=2, model=0): 150, ProcessCoord(pipe=37, data=3, model=0): 151, ProcessCoord(pipe=38, data=0, model=0): 152, ProcessCoord(pipe=38, data=1, model=0): 153, ProcessCoord(pipe=38, data=2, model=0): 154, ProcessCoord(pipe=38, data=3, model=0): 155, ProcessCoord(pipe=39, data=0, model=0): 156, ProcessCoord(pipe=39, data=1, model=0): 157, ProcessCoord(pipe=39, data=2, model=0): 158, ProcessCoord(pipe=39, data=3, model=0): 159, ProcessCoord(pipe=40, data=0, model=0): 160, ProcessCoord(pipe=40, data=1, model=0): 161, ProcessCoord(pipe=40, data=2, model=0): 162, ProcessCoord(pipe=40, data=3, model=0): 163, ProcessCoord(pipe=41, data=0, model=0): 164, ProcessCoord(pipe=41, data=1, model=0): 165, ProcessCoord(pipe=41, data=2, model=0): 166, ProcessCoord(pipe=41, data=3, model=0): 167, ProcessCoord(pipe=42, data=0, model=0): 168, ProcessCoord(pipe=42, data=1, model=0): 169, ProcessCoord(pipe=42, data=2, model=0): 170, ProcessCoord(pipe=42, data=3, model=0): 171, ProcessCoord(pipe=43, data=0, model=0): 172, ProcessCoord(pipe=43, data=1, model=0): 173, ProcessCoord(pipe=43, data=2, model=0): 174, ProcessCoord(pipe=43, data=3, model=0): 175, ProcessCoord(pipe=44, data=0, model=0): 176, ProcessCoord(pipe=44, data=1, model=0): 177, ProcessCoord(pipe=44, data=2, model=0): 178, ProcessCoord(pipe=44, data=3, model=0): 179, ProcessCoord(pipe=45, data=0, model=0): 180, ProcessCoord(pipe=45, data=1, model=0): 181, ProcessCoord(pipe=45, data=2, model=0): 182, ProcessCoord(pipe=45, data=3, model=0): 183, ProcessCoord(pipe=46, data=0, model=0): 184, ProcessCoord(pipe=46, data=1, model=0): 185, ProcessCoord(pipe=46, data=2, model=0): 186, ProcessCoord(pipe=46, data=3, model=0): 187, ProcessCoord(pipe=47, data=0, model=0): 188, ProcessCoord(pipe=47, data=1, model=0): 189, ProcessCoord(pipe=47, data=2, model=0): 190, ProcessCoord(pipe=47, data=3, model=0): 191, ProcessCoord(pipe=48, data=0, model=0): 192, ProcessCoord(pipe=48, data=1, model=0): 193, ProcessCoord(pipe=48, data=2, model=0): 194, ProcessCoord(pipe=48, data=3, model=0): 195, ProcessCoord(pipe=49, data=0, model=0): 196, ProcessCoord(pipe=49, data=1, model=0): 197, ProcessCoord(pipe=49, data=2, model=0): 198, ProcessCoord(pipe=49, data=3, model=0): 199, ProcessCoord(pipe=50, data=0, model=0): 200, ProcessCoord(pipe=50, data=1, model=0): 201, ProcessCoord(pipe=50, data=2, model=0): 202, ProcessCoord(pipe=50, data=3, model=0): 203, ProcessCoord(pipe=51, data=0, model=0): 204, ProcessCoord(pipe=51, data=1, model=0): 205, ProcessCoord(pipe=51, data=2, model=0): 206, ProcessCoord(pipe=51, data=3, model=0): 207, ProcessCoord(pipe=52, data=0, model=0): 208, ProcessCoord(pipe=52, data=1, model=0): 209, ProcessCoord(pipe=52, data=2, model=0): 210, ProcessCoord(pipe=52, data=3, model=0): 211, ProcessCoord(pipe=53, data=0, model=0): 212, ProcessCoord(pipe=53, data=1, model=0): 213, ProcessCoord(pipe=53, data=2, model=0): 214, ProcessCoord(pipe=53, data=3, model=0): 215, ProcessCoord(pipe=54, data=0, model=0): 216, ProcessCoord(pipe=54, data=1, model=0): 217, ProcessCoord(pipe=54, data=2, model=0): 218, ProcessCoord(pipe=54, data=3, model=0): 219, ProcessCoord(pipe=55, data=0, model=0): 220, ProcessCoord(pipe=55, data=1, model=0): 221, ProcessCoord(pipe=55, data=2, model=0): 222, ProcessCoord(pipe=55, data=3, model=0): 223, ProcessCoord(pipe=56, data=0, model=0): 224, ProcessCoord(pipe=56, data=1, model=0): 225, ProcessCoord(pipe=56, data=2, model=0): 226, ProcessCoord(pipe=56, data=3, model=0): 227, ProcessCoord(pipe=57, data=0, model=0): 228, ProcessCoord(pipe=57, data=1, model=0): 229, ProcessCoord(pipe=57, data=2, model=0): 230, ProcessCoord(pipe=57, data=3, model=0): 231, ProcessCoord(pipe=58, data=0, model=0): 232, ProcessCoord(pipe=58, data=1, model=0): 233, ProcessCoord(pipe=58, data=2, model=0): 234, ProcessCoord(pipe=58, data=3, model=0): 235, ProcessCoord(pipe=59, data=0, model=0): 236, ProcessCoord(pipe=59, data=1, model=0): 237, ProcessCoord(pipe=59, data=2, model=0): 238, ProcessCoord(pipe=59, data=3, model=0): 239, ProcessCoord(pipe=60, data=0, model=0): 240, ProcessCoord(pipe=60, data=1, model=0): 241, ProcessCoord(pipe=60, data=2, model=0): 242, ProcessCoord(pipe=60, data=3, model=0): 243, ProcessCoord(pipe=61, data=0, model=0): 244, ProcessCoord(pipe=61, data=1, model=0): 245, ProcessCoord(pipe=61, data=2, model=0): 246, ProcessCoord(pipe=61, data=3, model=0): 247, ProcessCoord(pipe=62, data=0, model=0): 248, ProcessCoord(pipe=62, data=1, model=0): 249, ProcessCoord(pipe=62, data=2, model=0): 250, ProcessCoord(pipe=62, data=3, model=0): 251, ProcessCoord(pipe=63, data=0, model=0): 252, ProcessCoord(pipe=63, data=1, model=0): 253, ProcessCoord(pipe=63, data=2, model=0): 254, ProcessCoord(pipe=63, data=3, model=0): 255, ProcessCoord(pipe=64, data=0, model=0): 256, ProcessCoord(pipe=64, data=1, model=0): 257, ProcessCoord(pipe=64, data=2, model=0): 258, ProcessCoord(pipe=64, data=3, model=0): 259, ProcessCoord(pipe=65, data=0, model=0): 260, ProcessCoord(pipe=65, data=1, model=0): 261, ProcessCoord(pipe=65, data=2, model=0): 262, ProcessCoord(pipe=65, data=3, model=0): 263, ProcessCoord(pipe=66, data=0, model=0): 264, ProcessCoord(pipe=66, data=1, model=0): 265, ProcessCoord(pipe=66, data=2, model=0): 266, ProcessCoord(pipe=66, data=3, model=0): 267, ProcessCoord(pipe=67, data=0, model=0): 268, ProcessCoord(pipe=67, data=1, model=0): 269, ProcessCoord(pipe=67, data=2, model=0): 270, ProcessCoord(pipe=67, data=3, model=0): 271, ProcessCoord(pipe=68, data=0, model=0): 272, ProcessCoord(pipe=68, data=1, model=0): 273, ProcessCoord(pipe=68, data=2, model=0): 274, ProcessCoord(pipe=68, data=3, model=0): 275, ProcessCoord(pipe=69, data=0, model=0): 276, ProcessCoord(pipe=69, data=1, model=0): 277, ProcessCoord(pipe=69, data=2, model=0): 278, ProcessCoord(pipe=69, data=3, model=0): 279, ProcessCoord(pipe=70, data=0, model=0): 280, ProcessCoord(pipe=70, data=1, model=0): 281, ProcessCoord(pipe=70, data=2, model=0): 282, ProcessCoord(pipe=70, data=3, model=0): 283, ProcessCoord(pipe=71, data=0, model=0): 284, ProcessCoord(pipe=71, data=1, model=0): 285, ProcessCoord(pipe=71, data=2, model=0): 286, ProcessCoord(pipe=71, data=3, model=0): 287} -[default0]:[2022-09-07 21:53:42,491] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer|embedding -[default0]:stage=0 layers=3 -[default0]: 0: _to_float16 -[default0]: 1: EmbeddingPipe -[default0]: 2: -[default0]:stage=1 layers=1 -[default0]: 3: ParallelTransformerLayerPipe -[default0]:stage=2 layers=1 -[default0]: 4: ParallelTransformerLayerPipe -[default0]:stage=3 layers=1 -[default0]: 5: ParallelTransformerLayerPipe -[default0]:stage=4 layers=1 -[default0]: 6: ParallelTransformerLayerPipe -[default0]:stage=5 layers=1 -[default0]: 7: ParallelTransformerLayerPipe -[default0]:stage=6 layers=1 -[default0]: 8: ParallelTransformerLayerPipe -[default0]:stage=7 layers=1 -[default0]: 9: ParallelTransformerLayerPipe -[default0]:stage=8 layers=1 -[default0]: 10: ParallelTransformerLayerPipe -[default0]:stage=9 layers=1 -[default0]: 11: ParallelTransformerLayerPipe -[default0]:stage=10 layers=1 -[default0]: 12: ParallelTransformerLayerPipe -[default0]:stage=11 layers=1 -[default0]: 13: ParallelTransformerLayerPipe -[default0]:stage=12 layers=1 -[default0]: 14: ParallelTransformerLayerPipe -[default0]:stage=13 layers=1 -[default0]: 15: ParallelTransformerLayerPipe -[default0]:stage=14 layers=1 -[default0]: 16: ParallelTransformerLayerPipe -[default0]:stage=15 layers=1 -[default0]: 17: ParallelTransformerLayerPipe -[default0]:stage=16 layers=1 -[default0]: 18: ParallelTransformerLayerPipe -[default0]:stage=17 layers=1 -[default0]: 19: ParallelTransformerLayerPipe -[default0]:stage=18 layers=1 -[default0]: 20: ParallelTransformerLayerPipe -[default0]:stage=19 layers=1 -[default0]: 21: ParallelTransformerLayerPipe -[default0]:stage=20 layers=1 -[default0]: 22: ParallelTransformerLayerPipe -[default0]:stage=21 layers=1 -[default0]: 23: ParallelTransformerLayerPipe -[default0]:stage=22 layers=1 -[default0]: 24: ParallelTransformerLayerPipe -[default0]:stage=23 layers=1 -[default0]: 25: ParallelTransformerLayerPipe -[default0]:stage=24 layers=1 -[default0]: 26: ParallelTransformerLayerPipe -[default0]:stage=25 layers=1 -[default0]: 27: ParallelTransformerLayerPipe -[default0]:stage=26 layers=1 -[default0]: 28: ParallelTransformerLayerPipe -[default0]:stage=27 layers=1 -[default0]: 29: ParallelTransformerLayerPipe -[default0]:stage=28 layers=1 -[default0]: 30: ParallelTransformerLayerPipe -[default0]:stage=29 layers=1 -[default0]: 31: ParallelTransformerLayerPipe -[default0]:stage=30 layers=1 -[default0]: 32: ParallelTransformerLayerPipe -[default0]:stage=31 layers=1 -[default0]: 33: ParallelTransformerLayerPipe -[default0]:stage=32 layers=1 -[default0]: 34: ParallelTransformerLayerPipe -[default0]:stage=33 layers=1 -[default0]: 35: ParallelTransformerLayerPipe -[default0]:stage=34 layers=1 -[default0]: 36: ParallelTransformerLayerPipe -[default0]:stage=35 layers=1 -[default0]: 37: ParallelTransformerLayerPipe -[default0]:stage=36 layers=1 -[default0]: 38: ParallelTransformerLayerPipe -[default0]:stage=37 layers=1 -[default0]: 39: ParallelTransformerLayerPipe -[default0]:stage=38 layers=1 -[default0]: 40: ParallelTransformerLayerPipe -[default0]:stage=39 layers=1 -[default0]: 41: ParallelTransformerLayerPipe -[default0]:stage=40 layers=1 -[default0]: 42: ParallelTransformerLayerPipe -[default0]:stage=41 layers=1 -[default0]: 43: ParallelTransformerLayerPipe -[default0]:stage=42 layers=1 -[default0]: 44: ParallelTransformerLayerPipe -[default0]:stage=43 layers=1 -[default0]: 45: ParallelTransformerLayerPipe -[default0]:stage=44 layers=1 -[default0]: 46: ParallelTransformerLayerPipe -[default0]:stage=45 layers=1 -[default0]: 47: ParallelTransformerLayerPipe -[default0]:stage=46 layers=1 -[default0]: 48: ParallelTransformerLayerPipe -[default0]:stage=47 layers=1 -[default0]: 49: ParallelTransformerLayerPipe -[default0]:stage=48 layers=1 -[default0]: 50: ParallelTransformerLayerPipe -[default0]:stage=49 layers=1 -[default0]: 51: ParallelTransformerLayerPipe -[default0]:stage=50 layers=1 -[default0]: 52: ParallelTransformerLayerPipe -[default0]:stage=51 layers=1 -[default0]: 53: ParallelTransformerLayerPipe -[default0]:stage=52 layers=1 -[default0]: 54: ParallelTransformerLayerPipe -[default0]:stage=53 layers=1 -[default0]: 55: ParallelTransformerLayerPipe -[default0]:stage=54 layers=1 -[default0]: 56: ParallelTransformerLayerPipe -[default0]:stage=55 layers=1 -[default0]: 57: ParallelTransformerLayerPipe -[default0]:stage=56 layers=1 -[default0]: 58: ParallelTransformerLayerPipe -[default0]:stage=57 layers=1 -[default0]: 59: ParallelTransformerLayerPipe -[default0]:stage=58 layers=1 -[default0]: 60: ParallelTransformerLayerPipe -[default0]:stage=59 layers=1 -[default0]: 61: ParallelTransformerLayerPipe -[default0]:stage=60 layers=1 -[default0]: 62: ParallelTransformerLayerPipe -[default0]:stage=61 layers=1 -[default0]: 63: ParallelTransformerLayerPipe -[default0]:stage=62 layers=1 -[default0]: 64: ParallelTransformerLayerPipe -[default0]:stage=63 layers=1 -[default0]: 65: ParallelTransformerLayerPipe -[default0]:stage=64 layers=1 -[default0]: 66: ParallelTransformerLayerPipe -[default0]:stage=65 layers=1 -[default0]: 67: ParallelTransformerLayerPipe -[default0]:stage=66 layers=1 -[default0]: 68: ParallelTransformerLayerPipe -[default0]:stage=67 layers=1 -[default0]: 69: ParallelTransformerLayerPipe -[default0]:stage=68 layers=1 -[default0]: 70: ParallelTransformerLayerPipe -[default0]:stage=69 layers=1 -[default0]: 71: ParallelTransformerLayerPipe -[default0]:stage=70 layers=3 -[default0]: 72: ParallelTransformerLayerPipe -[default0]: 73: undo -[default0]: 74: MixedFusedLayerNorm -[default0]:stage=71 layers=2 -[default0]: 75: EmbeddingPipe -[default0]: 76: float16_to_fp32 -[default0]: loss: CrossEntropy -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Emitting ninja build file /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113/utils/build.ninja... -[default4]:Building extension module utils... -[default4]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Loading extension module utils... -[default6]:Loading extension module utils... -[default7]:Loading extension module utils... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:ninja: no work to do. -[default4]:Loading extension module utils... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.144256591796875 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.12553071975708008 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.1258697509765625 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.12561821937561035 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.11437702178955078 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.14424610137939453 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.11481475830078125 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.14423775672912598 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.14424562454223633 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.10367536544799805 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.12550997734069824 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.13740921020507812 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.13734078407287598 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.13724255561828613 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.10312867164611816 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.13722920417785645 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.4815833568572998 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.4815683364868164 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.48157691955566406 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.48155736923217773 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.1220388412475586 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Loading extension module utils... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.12164592742919922 seconds -[default2]:Time to load utils op: 0.12125539779663086 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.10222482681274414 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.11589360237121582 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.10361599922180176 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.10221409797668457 seconds -[default7]:Loading extension module utils... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Time to load utils op: 0.10231280326843262 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.10237288475036621 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.12113785743713379 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.10226655006408691 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.10228466987609863 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.15076327323913574 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.15076327323913574 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.10216140747070312 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.15076613426208496 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.1507573127746582 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.10250139236450195 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.10232377052307129 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.10212182998657227 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.14462709426879883 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.14462566375732422 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.14455127716064453 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.10232806205749512 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.10243630409240723 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.11590933799743652 seconds -[default6]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.11588215827941895 seconds -[default6]:Time to load utils op: 0.11589312553405762 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.14453577995300293 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.10250544548034668 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.10247206687927246 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.10238385200500488 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.11750268936157227 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.10209035873413086 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.10257649421691895 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.13351750373840332 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.13353180885314941 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.21696758270263672 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.11749958992004395 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.11750459671020508 seconds -[default6]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.11749696731567383 seconds -[default6]:Time to load utils op: 0.10233139991760254 seconds -[default4]:Loading extension module utils... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.1330866813659668 seconds -[default4]:Time to load utils op: 0.10253143310546875 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.1018984317779541 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.10232353210449219 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.1335279941558838 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.11428093910217285 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.21654033660888672 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.11395430564880371 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.21679091453552246 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.10857176780700684 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.2167806625366211 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.10854792594909668 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.10906791687011719 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.10953664779663086 seconds -[default6]:Loading extension module utils... -[default7]:Loading extension module utils... -[default4]:Loading extension module utils... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.14474034309387207 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Loading extension module utils... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.10248827934265137 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.1447160243988037 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.10234808921813965 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.10237288475036621 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.10250616073608398 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.14473390579223633 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.1447286605834961 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Time to load utils op: 0.5585310459136963 seconds -[default5]:Time to load utils op: 0.55859375 seconds -[default7]:Time to load utils op: 0.5584471225738525 seconds -[default4]:Time to load utils op: 0.5597527027130127 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:[2022-09-07 21:53:44,254] [INFO] [utils.py:827:see_memory_usage] After Building Model -[default0]:[2022-09-07 21:53:44,255] [INFO] [utils.py:828:see_memory_usage] MA 6.7 GB Max_MA 6.7 GB CA 6.7 GB Max_CA 7 GB -[default0]:[2022-09-07 21:53:44,255] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.32 GB, percent = 6.2% -[default0]:setting training iterations to 3100 -[default0]:> learning rate decay style: constant -[default0]:DeepSpeed is enabled. -[default0]:[2022-09-07 21:53:44,256] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.1+8b2a6371, git-hash=8b2a6371, git-branch=master -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Emitting ninja build file /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113/utils/build.ninja... -[default0]:Building extension module utils... -[default0]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default0]:ninja: no work to do. -[default6]:Time to load utils op: 0.725006103515625 seconds -[default4]:Time to load utils op: 0.7247297763824463 seconds -[default7]:Time to load utils op: 0.7249598503112793 seconds -[default5]:Time to load utils op: 0.7247316837310791 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:[2022-09-07 21:53:44,982] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False -[default0]:[2022-09-07 21:53:44,982] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer -[default0]:[2022-09-07 21:53:44,982] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer -[default0]:[2022-09-07 21:53:44,982] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = {basic_optimizer.__class__.__name__} -[default0]:[2022-09-07 21:53:44,982] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer -[default0]:[2022-09-07 21:53:45,011] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer -[default0]:[2022-09-07 21:53:45,011] [INFO] [utils.py:828:see_memory_usage] MA 6.7 GB Max_MA 6.7 GB CA 6.7 GB Max_CA 7 GB -[default0]:[2022-09-07 21:53:45,011] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.48 GB, percent = 6.3% -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.9876301288604736 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.987699031829834 seconds -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.988112211227417 seconds -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.9874606132507324 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.1021180152893066 seconds -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.1478476524353027 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.1478374004364014 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.1478612422943115 seconds -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.1114583015441895 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.1022090911865234 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default1]:Time to load utils op: 1.1023378372192383 seconds -[default0]:Time to load utils op: 1.1113829612731934 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.1024210453033447 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.0908458232879639 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.1478495597839355 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.1040189266204834 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.1041057109832764 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.1039917469024658 seconds -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.0840938091278076 seconds -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.1040325164794922 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.0516505241394043 seconds -[default1]:Time to load utils op: 1.0518901348114014 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.0516705513000488 seconds -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.102121353149414 seconds -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.1514101028442383 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.1510040760040283 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.1511893272399902 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.1100969314575195 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.0907044410705566 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.0486085414886475 seconds -[default7]:Time to load utils op: 1.083829641342163 seconds -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.0840387344360352 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.048917293548584 seconds -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.0516784191131592 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.0515477657318115 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.1050026416778564 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.0516211986541748 seconds -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.1049246788024902 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.1047487258911133 seconds -[default0]:Time to load utils op: 1.104947805404663 seconds -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.0885958671569824 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.088928461074829 seconds -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.0841634273529053 seconds -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.104771614074707 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default0]:Time to load utils op: 1.1047840118408203 seconds -[default2]:Time to load utils op: 1.1046288013458252 seconds -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.1099822521209717 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.0890238285064697 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.1048712730407715 seconds -[default5]:Time to load utils op: 1.088886022567749 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.1098792552947998 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.0839481353759766 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.1101491451263428 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.0837063789367676 seconds -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.1022531986236572 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.1509881019592285 seconds -[default0]:Loading extension module utils... -[default3]:Loading extension module utils... -[default2]:Loading extension module utils... -[default1]:Loading extension module utils... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.1465623378753662 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.3220453262329102 seconds -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.0839409828186035 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.0839898586273193 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.103665828704834 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.1038641929626465 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.146538257598877 seconds -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.0831058025360107 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.0808486938476562 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.08282470703125 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.0831174850463867 seconds -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.103501796722412 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.1465656757354736 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.146569013595581 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.0861363410949707 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.1032874584197998 seconds -[default1]:Loading extension module utils... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.111372470855713 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.0812349319458008 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.1113684177398682 seconds -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.081291675567627 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.081347942352295 seconds -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.1017727851867676 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.101811408996582 seconds -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.1017732620239258 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.1018755435943604 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.5150058269500732 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.5148553848266602 seconds -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.515073299407959 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.1025757789611816 seconds -[default3]:Loading extension module utils... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.514925479888916 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.3220398426055908 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.3220446109771729 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.3220469951629639 seconds -[default5]:Loading extension module utils... -[default4]:Loading extension module utils... -[default2]:Loading extension module utils... -[default6]:Loading extension module utils... -[default4]:Loading extension module utils... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.086454153060913 seconds -[default5]:Time to load utils op: 1.0860939025878906 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default7]:Loading extension module utils... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.1026339530944824 seconds -[default7]:Time to load utils op: 1.0862820148468018 seconds -[default4]:Time to load utils op: 1.0863761901855469 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.102405309677124 seconds -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.1022582054138184 seconds -[default1]:Loading extension module utils... -[default4]:Loading extension module utils... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.2993035316467285 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.2857511043548584 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.3292887210845947 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.3280770778656006 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.3259754180908203 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.326582431793213 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.3263163566589355 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.325932264328003 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.328962802886963 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.3283514976501465 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.3061752319335938 seconds -[default3]:Loading extension module utils... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.308122158050537 seconds -[default3]:Time to load utils op: 1.3075063228607178 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.3052341938018799 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.2889392375946045 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0014095306396484375 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.3586382865905762 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.3172657489776611 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.3172650337219238 seconds -[default0]:Time to load utils op: 1.2775959968566895 seconds -[default1]:Time to load utils op: 1.2771248817443848 seconds -[default3]:Time to load utils op: 1.2767999172210693 seconds -[default2]:Time to load utils op: 1.2767059803009033 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.3172712326049805 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.2992994785308838 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.3172602653503418 seconds -[default1]:Time to load utils op: 1.299285650253296 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.3248074054718018 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.324962854385376 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.3248393535614014 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.286583423614502 seconds -[default3]:Time to load utils op: 1.2993052005767822 seconds -[default5]:Time to load utils op: 1.2992873191833496 seconds -[default4]:Time to load utils op: 1.2992708683013916 seconds -[default6]:Time to load utils op: 1.2992885112762451 seconds -[default2]:Time to load utils op: 1.299278736114502 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.2913975715637207 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.2859687805175781 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.2914085388183594 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.2913808822631836 seconds -[default4]:Time to load utils op: 1.3260581493377686 seconds -[default1]:Time to load utils op: 1.2863473892211914 seconds -[default4]:Time to load utils op: 1.2913987636566162 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007090568542480469 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0007693767547607422 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0011146068572998047 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0009953975677490234 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0010881423950195312 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0009810924530029297 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0008261203765869141 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.000583648681640625 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0009706020355224609 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005233287811279297 seconds -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005164146423339844 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0007622241973876953 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006496906280517578 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006487369537353516 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.000530242919921875 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0011200904846191406 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0008714199066162109 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0010747909545898438 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0013337135314941406 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.00064849853515625 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005819797515869141 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0006549358367919922 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.001016855239868164 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005772113800048828 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0007848739624023438 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0008344650268554688 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0010113716125488281 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006990432739257812 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.000637054443359375 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0007328987121582031 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0008814334869384766 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.001039743423461914 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005638599395751953 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007596015930175781 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0007731914520263672 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005791187286376953 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0007452964782714844 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0009801387786865234 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0009505748748779297 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0007660388946533203 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006532669067382812 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0007576942443847656 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007295608520507812 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006477832794189453 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0008482933044433594 seconds -[default2]:Emitting ninja build file /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113/utils/build.ninja... -[default2]:Building extension module utils... -[default2]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default2]:Loading extension module utils... -[default0]:Loading extension module utils... -[default1]:Loading extension module utils... -[default4]:Loading extension module utils... -[default7]:Loading extension module utils... -[default3]:Loading extension module utils... -[default6]:Loading extension module utils... -[default5]:Loading extension module utils... -[default0]:Loading extension module utils... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.7919232845306396 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.796729564666748 seconds -[default0]:Loading extension module utils... -[default5]:Loading extension module utils... -[default2]:Loading extension module utils... -[default4]:Loading extension module utils... -[default7]:Loading extension module utils... -[default1]:Loading extension module utils... -[default3]:Loading extension module utils... -[default6]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.7969517707824707 seconds -[default7]:Loading extension module utils... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.6759226322174072 seconds -[default6]:Loading extension module utils... -[default2]:Loading extension module utils... -[default3]:Loading extension module utils... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.6763851642608643 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.6761109828948975 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.7920939922332764 seconds -[default1]:Loading extension module utils... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.6606128215789795 seconds -[default1]:Time to load utils op: 1.660616397857666 seconds -[default0]:Loading extension module utils... -[default1]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.6690139770507812 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.6690399646759033 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.6692938804626465 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.6943738460540771 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.6950671672821045 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.6496119499206543 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.67844557762146 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.678468942642212 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.6784586906433105 seconds -[default3]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.6784627437591553 seconds -[default3]:Time to load utils op: 1.678466558456421 seconds -[default2]:ninja: no work to do. -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.6267340183258057 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.6475298404693604 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.6338071823120117 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.6475191116333008 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.6474967002868652 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.670168399810791 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.64963698387146 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.6496038436889648 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.6697421073913574 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.6700999736785889 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.6694848537445068 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.6606147289276123 seconds -[default4]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.6606216430664062 seconds -[default4]:Time to load utils op: 1.6606271266937256 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.6606338024139404 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.6495778560638428 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.6496143341064453 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.6496350765228271 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.6496119499206543 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.649609088897705 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.6475327014923096 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.6756911277770996 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.6752259731292725 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.6943504810333252 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.6761536598205566 seconds -[default6]:Loading extension module utils... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.6944689750671387 seconds -[default6]:Time to load utils op: 1.6753392219543457 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.680063009262085 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.660623550415039 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.6806306838989258 seconds -[default2]:Loading extension module utils... -[default1]:Loading extension module utils... -[default3]:Loading extension module utils... -[default7]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.6979491710662842 seconds -[default7]:Time to load utils op: 1.6969387531280518 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.6969668865203857 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.6971611976623535 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.6784462928771973 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.6798245906829834 seconds -[default0]:Loading extension module utils... -[default4]:Loading extension module utils... -[default5]:Loading extension module utils... -[default6]:Loading extension module utils... -[default7]:Loading extension module utils... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.6793019771575928 seconds -[default0]:Loading extension module utils... -[default2]:Loading extension module utils... -[default1]:Loading extension module utils... -[default4]:Loading extension module utils... -[default3]:Loading extension module utils... -[default7]:Loading extension module utils... -[default5]:Loading extension module utils... -[default6]:Loading extension module utils... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.763613224029541 seconds -[default2]:Loading extension module utils... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.7353293895721436 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.6606171131134033 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.709120273590088 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.7091033458709717 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.7629742622375488 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.7626762390136719 seconds -[default1]:Time to load utils op: 1.6877284049987793 seconds -[default0]:Time to load utils op: 1.6879758834838867 seconds -[default3]:Time to load utils op: 1.6875956058502197 seconds -[default7]:Time to load utils op: 1.7015511989593506 seconds -[default2]:Time to load utils op: 1.687666893005371 seconds -[default4]:Time to load utils op: 1.701538324356079 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.6799893379211426 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.6800899505615234 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.679323673248291 seconds -[default6]:Time to load utils op: 1.7015571594238281 seconds -[default5]:Time to load utils op: 1.7015445232391357 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.7485706806182861 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.748582124710083 seconds -[default0]:Time to load utils op: 1.6711628437042236 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.6796371936798096 seconds -[default0]:Time to load utils op: 1.675919771194458 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.7052171230316162 seconds -[default2]:Time to load utils op: 1.671165943145752 seconds -[default5]:Time to load utils op: 1.6711533069610596 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.705193042755127 seconds -[default4]:Time to load utils op: 1.6711704730987549 seconds -[default7]:Time to load utils op: 1.6711606979370117 seconds -[default3]:Time to load utils op: 1.6711483001708984 seconds -[default1]:Time to load utils op: 1.6711809635162354 seconds -[default6]:Time to load utils op: 1.6711759567260742 seconds -[default1]:Loading extension module utils... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.7134239673614502 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.712346076965332 seconds -[default1]:Time to load utils op: 1.7124443054199219 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.691190481185913 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.6911897659301758 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.6912007331848145 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.6911981105804443 seconds -[default4]:Loading extension module utils... -[default7]:Time to load utils op: 1.676360845565796 seconds -[default5]:Loading extension module utils... -[default6]:Time to load utils op: 1.6763522624969482 seconds -[default4]:Time to load utils op: 1.7041730880737305 seconds -[default2]:Time to load utils op: 1.6764864921569824 seconds -[default3]:Loading extension module utils... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.7030634880065918 seconds -[default5]:Time to load utils op: 1.7036235332489014 seconds -[default3]:Time to load utils op: 1.7052175998687744 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.7033586502075195 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.7052021026611328 seconds -[default3]:Time to load utils op: 1.6764719486236572 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.7485816478729248 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.7121074199676514 seconds -[default5]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.748584270477295 seconds -[default5]:Time to load utils op: 1.6938645839691162 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.6933209896087646 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.6943843364715576 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.6934382915496826 seconds -[default0]:Time to load utils op: 1.6784532070159912 seconds -[default1]:Time to load utils op: 1.6784632205963135 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.7489814758300781 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.7483506202697754 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.687913179397583 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.68790864944458 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.6879100799560547 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.687812328338623 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0004723072052001953 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005691051483154297 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.00044274330139160156 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.6864492893218994 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005376338958740234 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005283355712890625 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.6879229545593262 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.6865415573120117 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.7153804302215576 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0003516674041748047 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.7495677471160889 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005500316619873047 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0022537708282470703 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0022656917572021484 seconds -[default2]:Time to load utils op: 1.7332894802093506 seconds -[default1]:Time to load utils op: 1.7332653999328613 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.7482824325561523 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.6870288848876953 seconds -[default3]:Time to load utils op: 1.7332813739776611 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0004878044128417969 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005035400390625 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006229877471923828 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0004975795745849609 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.002340555191040039 seconds -[default0]:Time to load utils op: 1.7333083152770996 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.7091031074523926 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.7091162204742432 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.7091248035430908 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.7091035842895508 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.7091069221496582 seconds -[default4]:Time to load utils op: 1.7068297863006592 seconds -[default5]:Time to load utils op: 1.7061047554016113 seconds -[default6]:Time to load utils op: 1.7062227725982666 seconds -[default7]:Time to load utils op: 1.7060539722442627 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0022890567779541016 seconds -[default0]:Time to load utils op: 1.6849915981292725 seconds -[default2]:Time to load utils op: 1.684993028640747 seconds -[default1]:Time to load utils op: 1.6849935054779053 seconds -[default4]:Time to load utils op: 1.6849820613861084 seconds -[default3]:Time to load utils op: 1.6849863529205322 seconds -[default7]:Time to load utils op: 1.6849825382232666 seconds -[default6]:Time to load utils op: 1.6849756240844727 seconds -[default5]:Time to load utils op: 1.6849822998046875 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0003790855407714844 seconds -[default2]:Time to load utils op: 1.7091255187988281 seconds -[default0]:[2022-09-07 21:53:45,775] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 -[default0]:[2022-09-07 21:53:45,775] [INFO] [utils.py:828:see_memory_usage] MA 6.7 GB Max_MA 6.7 GB CA 6.7 GB Max_CA 7 GB -[default0]:[2022-09-07 21:53:45,775] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.48 GB, percent = 6.3% -[default0]:[2022-09-07 21:53:45,863] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 -[default0]:[2022-09-07 21:53:45,863] [INFO] [utils.py:828:see_memory_usage] MA 23.45 GB Max_MA 23.45 GB CA 25.12 GB Max_CA 25 GB -[default0]:[2022-09-07 21:53:45,863] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.48 GB, percent = 6.3% -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006284713745117188 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.00040221214294433594 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006628036499023438 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.000736236572265625 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0007088184356689453 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Time to load utils op: 0.0006725788116455078 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0007848739624023438 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0007486343383789062 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006425380706787109 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005691051483154297 seconds -[default0]:Time to load utils op: 0.0005388259887695312 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006959438323974609 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.000705718994140625 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0037610530853271484 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0033690929412841797 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005512237548828125 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006532669067382812 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0004343986511230469 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005428791046142578 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0006785392761230469 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.002305269241333008 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005090236663818359 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006530284881591797 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.002245187759399414 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005822181701660156 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005638599395751953 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006608963012695312 seconds -[default7]:Time to load utils op: 0.0005936622619628906 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.003180980682373047 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0004830360412597656 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default1]:Time to load utils op: 0.003220796585083008 seconds -[default2]:Time to load utils op: 0.00310516357421875 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006885528564453125 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006017684936523438 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.00045752525329589844 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005764961242675781 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006086826324462891 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006225109100341797 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0023288726806640625 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.00041174888610839844 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005631446838378906 seconds -[default1]:Time to load utils op: 0.0006372928619384766 seconds -[default7]:Time to load utils op: 0.0005567073822021484 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.00057220458984375 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0024297237396240234 seconds -[default3]:Time to load utils op: 0.002464771270751953 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Loading extension module utils... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005426406860351562 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0033147335052490234 seconds -[default3]:Time to load utils op: 0.0005807876586914062 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0020160675048828125 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.004025936126708984 seconds -[default6]:Loading extension module utils... -[default7]:Time to load utils op: 0.002210378646850586 seconds -[default6]:Time to load utils op: 0.002237558364868164 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.003640413284301758 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0031554698944091797 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0033721923828125 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0035338401794433594 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.003390789031982422 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005171298980712891 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005550384521484375 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0010514259338378906 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0012748241424560547 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.004149198532104492 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.002772808074951172 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006155967712402344 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006897449493408203 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005824565887451172 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0006642341613769531 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0007772445678710938 seconds -[default1]:Time to load utils op: 0.0006544589996337891 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0012781620025634766 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0012080669403076172 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0020627975463867188 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0011112689971923828 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0010187625885009766 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0018613338470458984 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0012629032135009766 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0017888545989990234 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0018796920776367188 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.001893758773803711 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007047653198242188 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0018868446350097656 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0007042884826660156 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006649494171142578 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.001707315444946289 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006105899810791016 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005602836608886719 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006072521209716797 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006780624389648438 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005629062652587891 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005700588226318359 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0007145404815673828 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005917549133300781 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0008282661437988281 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0008649826049804688 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006654262542724609 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006608963012695312 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0029697418212890625 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0007033348083496094 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006213188171386719 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005624294281005859 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006463527679443359 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006632804870605469 seconds -[default6]:Time to load utils op: 0.0006935596466064453 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006091594696044922 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006301403045654297 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0018372535705566406 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0033006668090820312 seconds -[default3]:Loading extension module utils... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default3]:Time to load utils op: 0.0008347034454345703 seconds -[default1]:Time to load utils op: 0.0008063316345214844 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.00113677978515625 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0007998943328857422 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006945133209228516 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006747245788574219 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0006361007690429688 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006458759307861328 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005688667297363281 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006589889526367188 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006053447723388672 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006282329559326172 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.000667572021484375 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0028007030487060547 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.002348184585571289 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Time to load utils op: 0.0026276111602783203 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0024437904357910156 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0025708675384521484 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.002386331558227539 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0027313232421875 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0025026798248291016 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006597042083740234 seconds -[default0]:[2022-09-07 21:53:45,888] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 -[default0]:[2022-09-07 21:53:45,888] [INFO] [utils.py:828:see_memory_usage] MA 23.45 GB Max_MA 23.45 GB CA 25.12 GB Max_CA 25 GB -[default0]:[2022-09-07 21:53:45,888] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.49 GB, percent = 6.3% -[default0]:[2022-09-07 21:53:45,913] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 -[default0]:[2022-09-07 21:53:45,914] [INFO] [utils.py:828:see_memory_usage] MA 23.45 GB Max_MA 23.45 GB CA 25.12 GB Max_CA 25 GB -[default0]:[2022-09-07 21:53:45,914] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.49 GB, percent = 6.3% -[default0]:[2022-09-07 21:53:45,938] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer -[default0]:[2022-09-07 21:53:45,939] [INFO] [utils.py:828:see_memory_usage] MA 23.45 GB Max_MA 23.45 GB CA 25.12 GB Max_CA 25 GB -[default0]:[2022-09-07 21:53:45,939] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.49 GB, percent = 6.3% -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005230903625488281 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0007736682891845703 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006804466247558594 seconds -[default0]:[2022-09-07 21:53:45,999] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer -[default0]:[2022-09-07 21:53:46,000] [INFO] [utils.py:828:see_memory_usage] MA 30.15 GB Max_MA 30.15 GB CA 31.82 GB Max_CA 32 GB -[default0]:[2022-09-07 21:53:46,000] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.49 GB, percent = 6.3% -[default0]:[2022-09-07 21:53:46,024] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer -[default0]:[2022-09-07 21:53:46,025] [INFO] [utils.py:828:see_memory_usage] MA 30.15 GB Max_MA 30.15 GB CA 31.82 GB Max_CA 32 GB -[default0]:[2022-09-07 21:53:46,025] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.49 GB, percent = 6.3% -[default0]:[2022-09-07 21:53:46,025] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam -[default0]:[2022-09-07 21:53:46,025] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler -[default0]:[2022-09-07 21:53:46,025] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = -[default0]:[2022-09-07 21:53:46,025] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[2e-05, 2e-05], mom=[(0.9, 0.95), (0.9, 0.95)] -[default0]:[2022-09-07 21:53:46,025] [INFO] [config.py:987:print] DeepSpeedEngine configuration: -[default0]:[2022-09-07 21:53:46,025] [INFO] [config.py:991:print] activation_checkpointing_config { -[default0]: "partition_activations": false, -[default0]: "contiguous_memory_optimization": false, -[default0]: "cpu_checkpointing": false, -[default0]: "number_checkpoints": null, -[default0]: "synchronize_checkpoint_boundary": false, -[default0]: "profile": false -[default0]:} -[default0]:[2022-09-07 21:53:46,025] [INFO] [config.py:991:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} -[default0]:[2022-09-07 21:53:46,025] [INFO] [config.py:991:print] amp_enabled .................. False -[default0]:[2022-09-07 21:53:46,025] [INFO] [config.py:991:print] amp_params ................... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] autotuning_config ............ { -[default0]: "enabled": false, -[default0]: "start_step": null, -[default0]: "end_step": null, -[default0]: "metric_path": null, -[default0]: "arg_mappings": null, -[default0]: "metric": "throughput", -[default0]: "model_info": null, -[default0]: "results_dir": null, -[default0]: "exps_dir": null, -[default0]: "overwrite": true, -[default0]: "fast": true, -[default0]: "start_profile_step": 3, -[default0]: "end_profile_step": 5, -[default0]: "tuner_type": "gridsearch", -[default0]: "tuner_early_stopping": 5, -[default0]: "tuner_num_trials": 50, -[default0]: "model_info_path": null, -[default0]: "mp_size": 1, -[default0]: "max_train_batch_size": null, -[default0]: "min_train_batch_size": 1, -[default0]: "max_train_micro_batch_size_per_gpu": 1.024000e+03, -[default0]: "min_train_micro_batch_size_per_gpu": 1, -[default0]: "num_tuning_micro_batch_sizes": 3 -[default0]:} -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] bfloat16_enabled ............. True -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] checkpoint_tag_validation_enabled True -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] checkpoint_tag_validation_fail False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] comms_config ................. -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] communication_data_type ...... None -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] curriculum_enabled ........... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] curriculum_params ............ False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] dataloader_drop_last ......... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] disable_allgather ............ False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] dump_state ................... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] dynamic_loss_scale_args ...... None -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] eigenvalue_enabled ........... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] eigenvalue_gas_boundary_resolution 1 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] eigenvalue_layer_name ........ bert.encoder.layer -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] eigenvalue_layer_num ......... 0 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] eigenvalue_max_iter .......... 100 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] eigenvalue_stability ......... 1e-06 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] eigenvalue_tol ............... 0.01 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] eigenvalue_verbose ........... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] elasticity_enabled ........... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] flops_profiler_config ........ { -[default0]: "enabled": false, -[default0]: "profile_step": 1, -[default0]: "module_depth": -1, -[default0]: "top_modules": 1, -[default0]: "detailed": true, -[default0]: "output_file": null -[default0]:} -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] fp16_auto_cast ............... None -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] fp16_enabled ................. False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] fp16_master_weights_and_gradients False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] global_rank .................. 0 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] gradient_accumulation_steps .. 512 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] gradient_clipping ............ 1.0 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] gradient_predivide_factor .... 1.0 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] initial_dynamic_scale ........ 1 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] load_universal_checkpoint .... True -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] loss_scale ................... 1.0 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] memory_breakdown ............. False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] monitor_config ............... -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] nebula_config ................ { -[default0]: "enabled": false, -[default0]: "persistent_storage_path": null, -[default0]: "persistent_time_interval": 100, -[default0]: "num_of_version_in_retention": 2, -[default0]: "enable_nebula_load": true, -[default0]: "load_path": null -[default0]:} -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] optimizer_legacy_fusion ...... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] optimizer_name ............... None -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] optimizer_params ............. None -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] pld_enabled .................. False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] pld_params ................... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] prescale_gradients ........... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] scheduler_name ............... None -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] scheduler_params ............. None -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] sparse_attention ............. None -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] sparse_gradients_enabled ..... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] steps_per_print .............. 2000 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] train_batch_size ............. 2048 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] train_micro_batch_size_per_gpu 1 -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] wall_clock_breakdown ......... False -[default0]:[2022-09-07 21:53:46,026] [INFO] [config.py:991:print] world_size ................... 4 -[default0]:[2022-09-07 21:53:46,027] [INFO] [config.py:991:print] zero_allow_untested_optimizer False -[default0]:[2022-09-07 21:53:46,027] [INFO] [config.py:991:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False -[default0]:[2022-09-07 21:53:46,027] [INFO] [config.py:991:print] zero_enabled ................. False -[default0]:[2022-09-07 21:53:46,027] [INFO] [config.py:991:print] zero_optimization_stage ...... 0 -[default0]:[2022-09-07 21:53:46,027] [INFO] [config.py:976:print_user_config] json = { -[default0]: "train_micro_batch_size_per_gpu": 1, -[default0]: "train_batch_size": 2.048000e+03, -[default0]: "gradient_clipping": 1.0, -[default0]: "zero_optimization": { -[default0]: "stage": 0 -[default0]: }, -[default0]: "bf16": { -[default0]: "enabled": true -[default0]: }, -[default0]: "steps_per_print": 2.000000e+03, -[default0]: "wall_clock_breakdown": false, -[default0]: "checkpoint": { -[default0]: "load_universal": true -[default0]: } -[default0]:} -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0004684925079345703 seconds -[default0]:[2022-09-07 21:53:46,027] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=512 micro_batch_size=1 -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=144 STAGE=36 LAYERS=1 [38, 39) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=148 STAGE=37 LAYERS=1 [39, 40) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=248 STAGE=62 LAYERS=1 [64, 65) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=176 STAGE=44 LAYERS=1 [46, 47) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=88 STAGE=22 LAYERS=1 [24, 25) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=92 STAGE=23 LAYERS=1 [25, 26) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=104 STAGE=26 LAYERS=1 [28, 29) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=216 STAGE=54 LAYERS=1 [56, 57) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=264 STAGE=66 LAYERS=1 [68, 69) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=268 STAGE=67 LAYERS=1 [69, 70) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=112 STAGE=28 LAYERS=1 [30, 31) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=32 STAGE=8 LAYERS=1 [10, 11) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=72 STAGE=18 LAYERS=1 [20, 21) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=80 STAGE=20 LAYERS=1 [22, 23) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=116 STAGE=29 LAYERS=1 [31, 32) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=272 STAGE=68 LAYERS=1 [70, 71) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=120 STAGE=30 LAYERS=1 [32, 33) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=252 STAGE=63 LAYERS=1 [65, 66) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=180 STAGE=45 LAYERS=1 [47, 48) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=284 STAGE=71 LAYERS=2 [75, 77) STAGE_PARAMS=3596615680 (3596.616M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=36 STAGE=9 LAYERS=1 [11, 12) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=84 STAGE=21 LAYERS=1 [23, 24) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=220 STAGE=55 LAYERS=1 [57, 58) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=124 STAGE=31 LAYERS=1 [33, 34) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=152 STAGE=38 LAYERS=1 [40, 41) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=16 STAGE=4 LAYERS=1 [6, 7) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=192 STAGE=48 LAYERS=1 [50, 51) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=172 STAGE=43 LAYERS=1 [45, 46) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=168 STAGE=42 LAYERS=1 [44, 45) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=136 STAGE=34 LAYERS=1 [36, 37) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=140 STAGE=35 LAYERS=1 [37, 38) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=24 STAGE=6 LAYERS=1 [8, 9) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=76 STAGE=19 LAYERS=1 [21, 22) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=156 STAGE=39 LAYERS=1 [41, 42) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=64 STAGE=16 LAYERS=1 [18, 19) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=232 STAGE=58 LAYERS=1 [60, 61) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=196 STAGE=49 LAYERS=1 [51, 52) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=236 STAGE=59 LAYERS=1 [61, 62) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=132 STAGE=33 LAYERS=1 [35, 36) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=280 STAGE=70 LAYERS=3 [72, 75) STAGE_PARAMS=2466465792 (2466.466M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=96 STAGE=24 LAYERS=1 [26, 27) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=28 STAGE=7 LAYERS=1 [9, 10) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=256 STAGE=64 LAYERS=1 [66, 67) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=20 STAGE=5 LAYERS=1 [7, 8) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=8 STAGE=2 LAYERS=1 [4, 5) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=260 STAGE=65 LAYERS=1 [67, 68) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=208 STAGE=52 LAYERS=1 [54, 55) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=128 STAGE=32 LAYERS=1 [34, 35) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=276 STAGE=69 LAYERS=1 [71, 72) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=68 STAGE=17 LAYERS=1 [19, 20) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=56 STAGE=14 LAYERS=1 [16, 17) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=60 STAGE=15 LAYERS=1 [17, 18) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=212 STAGE=53 LAYERS=1 [55, 56) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=48 STAGE=12 LAYERS=1 [14, 15) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=200 STAGE=50 LAYERS=1 [52, 53) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,704] [INFO] [engine.py:145:__init__] RANK=52 STAGE=13 LAYERS=1 [15, 16) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=204 STAGE=51 LAYERS=1 [53, 54) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=160 STAGE=40 LAYERS=1 [42, 43) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=164 STAGE=41 LAYERS=1 [43, 44) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=44 STAGE=11 LAYERS=1 [13, 14) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=184 STAGE=46 LAYERS=1 [48, 49) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=40 STAGE=10 LAYERS=1 [12, 13) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=188 STAGE=47 LAYERS=1 [49, 50) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=224 STAGE=56 LAYERS=1 [58, 59) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=240 STAGE=60 LAYERS=1 [62, 63) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=100 STAGE=25 LAYERS=1 [27, 28) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=12 STAGE=3 LAYERS=1 [5, 6) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=3 [0, 3) STAGE_PARAMS=3596644352 (3596.644M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=108 STAGE=27 LAYERS=1 [29, 30) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=228 STAGE=57 LAYERS=1 [59, 60) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=244 STAGE=61 LAYERS=1 [63, 64) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 21:53:46,698] [INFO] [engine.py:145:__init__] RANK=4 STAGE=1 LAYERS=1 [3, 4) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default3]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 21:53:47,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 21:53:47,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 21:53:47,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 21:53:47,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 21:53:47,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 21:53:47,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 21:53:47,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 21:53:47,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 21:53:47,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 21:53:47,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 21:53:47,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 21:53:56,814] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 183 -[default6]:[2022-09-07 21:53:56,813] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 182 -[default4]:[2022-09-07 21:53:57,704] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 180 -[default5]:[2022-09-07 21:53:57,706] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 181 -[default3]:[2022-09-07 21:53:57,871] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 179 -[default0]:[2022-09-07 21:53:57,987] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 176 -[default1]:[2022-09-07 21:53:57,990] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 177 -[default7]:[2022-09-07 21:53:58,008] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 271 -[default6]:[2022-09-07 21:53:58,018] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 270 -[default7]:[2022-09-07 21:53:58,791] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 103 -[default2]:[2022-09-07 21:53:59,066] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 178 -[default0]:[2022-09-07 21:53:59,171] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 264 -[default1]:[2022-09-07 21:53:59,206] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 265 -[default5]:[2022-09-07 21:53:59,447] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 269 -[default4]:[2022-09-07 21:53:59,448] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 268 -[default3]:[2022-09-07 21:53:59,554] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 99 -[default2]:[2022-09-07 21:53:59,759] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 266 -[default3]:[2022-09-07 21:53:59,836] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 283 -[default3]:[2022-09-07 21:53:59,772] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 267 -[default3]:[2022-09-07 21:53:59,887] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 115 -[default7]:[2022-09-07 21:54:00,082] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 71 -[default7]:[2022-09-07 21:54:00,464] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 119 -[default3]:[2022-09-07 21:54:00,399] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 131 -[default4]:[2022-09-07 21:54:00,768] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 276 -[default3]:[2022-09-07 21:54:00,768] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 107 -[default5]:[2022-09-07 21:54:00,780] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 277 -[default3]:[2022-09-07 21:54:00,848] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 123 -[default3]:[2022-09-07 21:54:00,808] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 67 -[default0]:[2022-09-07 21:54:01,069] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 104 -[default3]:[2022-09-07 21:54:01,002] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 275 -[default1]:[2022-09-07 21:54:01,045] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 105 -[default3]:[2022-09-07 21:54:01,055] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 243 -[default6]:[2022-09-07 21:54:01,165] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 110 -[default7]:[2022-09-07 21:54:01,170] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 111 -[default0]:[2022-09-07 21:54:01,233] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 64 -[default1]:[2022-09-07 21:54:01,241] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 65 -[default3]:[2022-09-07 21:54:01,428] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 91 -[default2]:[2022-09-07 21:54:01,404] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 90 -[default3]:[2022-09-07 21:54:01,426] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 11 -[default7]:[2022-09-07 21:54:01,430] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 247 -[default2]:[2022-09-07 21:54:01,428] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 10 -[default4]:[2022-09-07 21:54:01,489] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 244 -[default5]:[2022-09-07 21:54:01,500] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 245 -[default7]:[2022-09-07 21:54:01,624] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 223 -[default0]:[2022-09-07 21:54:01,754] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 120 -[default1]:[2022-09-07 21:54:01,859] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 281 -[default0]:[2022-09-07 21:54:01,840] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 272 -[default1]:[2022-09-07 21:54:01,849] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 273 -[default1]:[2022-09-07 21:54:01,864] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 121 -[default7]:[2022-09-07 21:54:01,862] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 95 -[default0]:[2022-09-07 21:54:01,819] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 280 -[default7]:[2022-09-07 21:54:01,929] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 127 -[default2]:[2022-09-07 21:54:01,902] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 282 -[default7]:[2022-09-07 21:54:02,016] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 279 -[default7]:[2022-09-07 21:54:02,024] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 79 -[default3]:[2022-09-07 21:54:02,056] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 51 -[default4]:[2022-09-07 21:54:02,028] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 12 -[default5]:[2022-09-07 21:54:02,039] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 13 -[default4]:[2022-09-07 21:54:02,169] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 116 -[default5]:[2022-09-07 21:54:02,123] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 109 -[default7]:[2022-09-07 21:54:02,169] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 55 -[default4]:[2022-09-07 21:54:02,120] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 108 -[default0]:[2022-09-07 21:54:02,224] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 112 -[default1]:[2022-09-07 21:54:02,227] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 113 -[default5]:[2022-09-07 21:54:02,183] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 117 -[default3]:[2022-09-07 21:54:02,275] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 171 -[default3]:[2022-09-07 21:54:02,282] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 187 -[default2]:[2022-09-07 21:54:02,306] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 82 -[default3]:[2022-09-07 21:54:02,308] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 83 -[default0]:[2022-09-07 21:54:02,330] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 240 -[default1]:[2022-09-07 21:54:02,327] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 241 -[default4]:[2022-09-07 21:54:02,424] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 172 -[default7]:[2022-09-07 21:54:02,480] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 135 -[default3]:[2022-09-07 21:54:02,398] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 155 -[default5]:[2022-09-07 21:54:02,427] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 173 -[default2]:[2022-09-07 21:54:02,410] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 154 -[default3]:[2022-09-07 21:54:02,472] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 43 -[default2]:[2022-09-07 21:54:02,476] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 42 -[default7]:[2022-09-07 21:54:02,606] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 175 -[default6]:[2022-09-07 21:54:02,602] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 174 -[default0]:[2022-09-07 21:54:02,731] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 192 -[default1]:[2022-09-07 21:54:02,744] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 193 -[default5]:[2022-09-07 21:54:02,726] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 125 -[default4]:[2022-09-07 21:54:02,715] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 124 -[default3]:[2022-09-07 21:54:02,702] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 203 -[default5]:[2022-09-07 21:54:02,789] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 69 -[default7]:[2022-09-07 21:54:02,795] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 15 -[default4]:[2022-09-07 21:54:02,790] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 68 -[default0]:[2022-09-07 21:54:02,954] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 88 -[default1]:[2022-09-07 21:54:02,941] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 89 -[default7]:[2022-09-07 21:54:02,895] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 39 -[default1]:[2022-09-07 21:54:02,940] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 73 -[default0]:[2022-09-07 21:54:02,930] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 72 -[default0]:[2022-09-07 21:54:02,957] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 40 -[default1]:[2022-09-07 21:54:02,953] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 41 -[default7]:[2022-09-07 21:54:03,004] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 215 -[default6]:[2022-09-07 21:54:02,987] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 214 -[default3]:[2022-09-07 21:54:03,028] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 227 -[default7]:[2022-09-07 21:54:03,042] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 231 -[default7]:[2022-09-07 21:54:03,126] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 87 -[default3]:[2022-09-07 21:54:03,130] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 259 -[default0]:[2022-09-07 21:54:03,157] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 96 -[default7]:[2022-09-07 21:54:03,091] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 199 -[default1]:[2022-09-07 21:54:03,175] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 97 -[default7]:[2022-09-07 21:54:03,112] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 159 -[default0]:[2022-09-07 21:54:03,108] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 8 -[default1]:[2022-09-07 21:54:03,112] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 9 -[default6]:[2022-09-07 21:54:03,111] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 14 -[default3]:[2022-09-07 21:54:03,233] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 211 -[default7]:[2022-09-07 21:54:03,257] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 63 -[default2]:[2022-09-07 21:54:03,224] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 106 -[default6]:[2022-09-07 21:54:03,261] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 246 -[default2]:[2022-09-07 21:54:03,327] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 274 -[default5]:[2022-09-07 21:54:03,307] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 213 -[default7]:[2022-09-07 21:54:03,379] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 207 -[default7]:[2022-09-07 21:54:03,300] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 47 -[default4]:[2022-09-07 21:54:03,307] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 212 -[default5]:[2022-09-07 21:54:03,390] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 5 -[default4]:[2022-09-07 21:54:03,325] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 4 -[default4]:[2022-09-07 21:54:03,414] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 92 -[default2]:[2022-09-07 21:54:03,445] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 114 -[default0]:[2022-09-07 21:54:03,398] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 168 -[default1]:[2022-09-07 21:54:03,406] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 169 -[default2]:[2022-09-07 21:54:03,422] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 194 -[default5]:[2022-09-07 21:54:03,412] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 93 -[default3]:[2022-09-07 21:54:03,412] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 195 -[default6]:[2022-09-07 21:54:03,443] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 126 -[default2]:[2022-09-07 21:54:03,392] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 242 -[default6]:[2022-09-07 21:54:03,506] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 118 -[default2]:[2022-09-07 21:54:03,544] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 74 -[default3]:[2022-09-07 21:54:03,564] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 75 -[default4]:[2022-09-07 21:54:03,513] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 196 -[default5]:[2022-09-07 21:54:03,519] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 197 -[default6]:[2022-09-07 21:54:03,577] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 278 -[default5]:[2022-09-07 21:54:03,519] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 101 -[default4]:[2022-09-07 21:54:03,511] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 100 -[default7]:[2022-09-07 21:54:03,579] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 151 -[default4]:[2022-09-07 21:54:03,590] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 132 -[default2]:[2022-09-07 21:54:03,651] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 66 -[default5]:[2022-09-07 21:54:03,619] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 133 -[default3]:[2022-09-07 21:54:03,711] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 27 -[default2]:[2022-09-07 21:54:03,701] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 122 -[default6]:[2022-09-07 21:54:03,796] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 6 -[default7]:[2022-09-07 21:54:03,799] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 7 -[default0]:[2022-09-07 21:54:03,845] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 80 -[default1]:[2022-09-07 21:54:03,828] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 81 -[default6]:[2022-09-07 21:54:03,837] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 78 -[default5]:[2022-09-07 21:54:03,842] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 157 -[default6]:[2022-09-07 21:54:03,821] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 102 -[default6]:[2022-09-07 21:54:03,847] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 70 -[default4]:[2022-09-07 21:54:03,836] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 156 -[default3]:[2022-09-07 21:54:03,833] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 19 -[default0]:[2022-09-07 21:54:03,805] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 184 -[default1]:[2022-09-07 21:54:03,810] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 185 -[default1]:[2022-09-07 21:54:03,921] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 217 -[default0]:[2022-09-07 21:54:03,915] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 216 -[default3]:[2022-09-07 21:54:03,914] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 139 -[default3]:[2022-09-07 21:54:03,967] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 219 -[default2]:[2022-09-07 21:54:03,934] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 218 -[default0]:[2022-09-07 21:54:03,972] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 128 -[default3]:[2022-09-07 21:54:04,056] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 251 -[default2]:[2022-09-07 21:54:04,057] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 98 -[default4]:[2022-09-07 21:54:04,078] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 220 -[default1]:[2022-09-07 21:54:03,988] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 129 -[default6]:[2022-09-07 21:54:04,036] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 198 -[default5]:[2022-09-07 21:54:04,050] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 61 -[default4]:[2022-09-07 21:54:04,043] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 60 -[default5]:[2022-09-07 21:54:04,086] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 45 -[default4]:[2022-09-07 21:54:04,077] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 44 -[default6]:[2022-09-07 21:54:04,128] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 94 -[default2]:[2022-09-07 21:54:04,097] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 170 -[default7]:[2022-09-07 21:54:04,094] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 263 -[default5]:[2022-09-07 21:54:04,090] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 221 -[default3]:[2022-09-07 21:54:04,245] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 147 -[default1]:[2022-09-07 21:54:04,251] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 249 -[default0]:[2022-09-07 21:54:04,234] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 248 -[default4]:[2022-09-07 21:54:04,241] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 84 -[default2]:[2022-09-07 21:54:04,225] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 34 -[default5]:[2022-09-07 21:54:04,242] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 85 -[default3]:[2022-09-07 21:54:04,240] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 35 -[default6]:[2022-09-07 21:54:04,196] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 134 -[default7]:[2022-09-07 21:54:04,203] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 167 -[default7]:[2022-09-07 21:54:04,240] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 191 -[default4]:[2022-09-07 21:54:04,375] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 28 -[default5]:[2022-09-07 21:54:04,296] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 29 -[default2]:[2022-09-07 21:54:04,423] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 130 -[default5]:[2022-09-07 21:54:04,418] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 261 -[default4]:[2022-09-07 21:54:04,408] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 260 -[default0]:[2022-09-07 21:54:04,420] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 208 -[default1]:[2022-09-07 21:54:04,426] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 209 -[default5]:[2022-09-07 21:54:04,427] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 205 -[default3]:[2022-09-07 21:54:04,435] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 59 -[default6]:[2022-09-07 21:54:04,417] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 46 -[default4]:[2022-09-07 21:54:04,424] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 204 -[default2]:[2022-09-07 21:54:04,431] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 50 -[default0]:[2022-09-07 21:54:04,549] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 136 -[default1]:[2022-09-07 21:54:04,527] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 137 -[default0]:[2022-09-07 21:54:04,507] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 16 -[default1]:[2022-09-07 21:54:04,506] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 17 -[default6]:[2022-09-07 21:54:04,505] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 222 -[default7]:[2022-09-07 21:54:04,517] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 31 -[default6]:[2022-09-07 21:54:04,509] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 30 -[default5]:[2022-09-07 21:54:04,585] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 189 -[default4]:[2022-09-07 21:54:04,582] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 188 -[default0]:[2022-09-07 21:54:04,520] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 224 -[default5]:[2022-09-07 21:54:04,591] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 77 -[default4]:[2022-09-07 21:54:04,615] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 76 -[default7]:[2022-09-07 21:54:04,602] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 143 -[default1]:[2022-09-07 21:54:04,650] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 225 -[default6]:[2022-09-07 21:54:04,717] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 62 -[default7]:[2022-09-07 21:54:04,778] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 23 -[default1]:[2022-09-07 21:54:04,825] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 145 -[default0]:[2022-09-07 21:54:04,819] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 144 -[default3]:[2022-09-07 21:54:04,864] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 163 -[default0]:[2022-09-07 21:54:04,885] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 152 -[default0]:[2022-09-07 21:54:04,896] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 24 -[default1]:[2022-09-07 21:54:04,975] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 25 -[default1]:[2022-09-07 21:54:04,926] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 153 -[default0]:[2022-09-07 21:54:04,958] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 56 -[default1]:[2022-09-07 21:54:04,966] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 57 -[default6]:[2022-09-07 21:54:04,993] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 86 -[default4]:[2022-09-07 21:54:05,076] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 36 -[default2]:[2022-09-07 21:54:05,074] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 210 -[default6]:[2022-09-07 21:54:05,045] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 158 -[default4]:[2022-09-07 21:54:05,038] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 20 -[default5]:[2022-09-07 21:54:05,054] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 21 -[default5]:[2022-09-07 21:54:05,099] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 37 -[default6]:[2022-09-07 21:54:05,168] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 262 -[default2]:[2022-09-07 21:54:05,168] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 26 -[default2]:[2022-09-07 21:54:05,198] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 18 -[default5]:[2022-09-07 21:54:05,246] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 229 -[default4]:[2022-09-07 21:54:05,234] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 228 -[default0]:[2022-09-07 21:54:05,381] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 256 -[default0]:[2022-09-07 21:54:05,369] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 200 -[default2]:[2022-09-07 21:54:05,378] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 186 -[default6]:[2022-09-07 21:54:05,373] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 190 -[default1]:[2022-09-07 21:54:05,375] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 201 -[default0]:[2022-09-07 21:54:05,391] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 32 -[default2]:[2022-09-07 21:54:05,456] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 250 -[default1]:[2022-09-07 21:54:05,388] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 33 -[default7]:[2022-09-07 21:54:05,465] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 255 -[default1]:[2022-09-07 21:54:05,390] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 257 -[default4]:[2022-09-07 21:54:05,456] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 52 -[default2]:[2022-09-07 21:54:05,435] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 58 -[default6]:[2022-09-07 21:54:05,427] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 230 -[default5]:[2022-09-07 21:54:05,463] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 53 -[default2]:[2022-09-07 21:54:05,430] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 226 -[default6]:[2022-09-07 21:54:05,487] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 38 -[default1]:[2022-09-07 21:54:05,572] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 49 -[default0]:[2022-09-07 21:54:05,569] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 48 -[default2]:[2022-09-07 21:54:05,671] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 138 -[default6]:[2022-09-07 21:54:05,656] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 54 -[default0]:[2022-09-07 21:54:05,678] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 160 -[default4]:[2022-09-07 21:54:05,716] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 148 -[default2]:[2022-09-07 21:54:05,750] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 146 -[default5]:[2022-09-07 21:54:05,731] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 149 -[default5]:[2022-09-07 21:54:05,737] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 253 -[default4]:[2022-09-07 21:54:05,737] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 252 -[default1]:[2022-09-07 21:54:05,723] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 161 -[default2]:[2022-09-07 21:54:05,780] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 202 -[default6]:[2022-09-07 21:54:05,743] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 22 -[default4]:[2022-09-07 21:54:05,797] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 140 -[default5]:[2022-09-07 21:54:05,801] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 141 -[default2]:[2022-09-07 21:54:05,856] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 258 -[default6]:[2022-09-07 21:54:05,964] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 142 -[default2]:[2022-09-07 21:54:05,961] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 162 -[default6]:[2022-09-07 21:54:05,935] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 206 -[default6]:[2022-09-07 21:54:06,049] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 254 -[default3]:[2022-09-07 21:54:06,164] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 235 -[default1]:[2022-09-07 21:54:06,168] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 233 -[default0]:[2022-09-07 21:54:06,160] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 232 -[default5]:[2022-09-07 21:54:06,204] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 165 -[default4]:[2022-09-07 21:54:06,199] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 164 -[default6]:[2022-09-07 21:54:06,330] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 150 -[default4]:[2022-09-07 21:54:06,400] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 236 -[default7]:[2022-09-07 21:54:06,480] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 239 -[default5]:[2022-09-07 21:54:06,450] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 237 -[default6]:[2022-09-07 21:54:06,579] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 238 -[default6]:[2022-09-07 21:54:06,523] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 166 -[default2]:[2022-09-07 21:54:06,632] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 234 -[default5]:[2022-09-07 21:54:08,249] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 285 -[default4]:[2022-09-07 21:54:08,225] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 284 -[default7]:[2022-09-07 21:54:08,352] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 287 -[default6]:[2022-09-07 21:54:08,329] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 286 -[default3]:[2022-09-07 21:54:08,945] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 3 -[default2]:[2022-09-07 21:54:09,014] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 2 -[default7]:time (ms) | load-checkpoint: 21450.31 -[default1]:[2022-09-07 21:54:09,019] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 1 -[default0]:[2022-09-07 21:54:09,029] [INFO] [engine.py:2763:_load_zero_checkpoint] loaded universal zero checkpoints from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step0_universal for rank 0 -[default0]:could not find arguments in the checkpoint ... -[default0]: checkpoint version 3.0 -[default0]: successfully loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq at iteration 0 -[default0]:estimated model parameters: 258.958393344 -[default0]:estimated model parameters without embeddings: 0.002064384 -[default0]:/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/utils.py:365: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings -[default0]: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") -[default0]:[after model, optimizer, and learning rate scheduler are built] datetime: 2022-09-07 21:54:09 -[default0]:> building train, validation, and test datasets ... -[default0]: > datasets target sizes (minimum size): -[default0]: train: 6348800 -[default0]: validation: 26624 -[default0]: test: 2048 -[default0]:> building train, validation, and test datasets for T0 ... -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.327884 seconds -[default0]: number of documents: 32740750 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 32740750) total of 32740750 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.050978 seconds -[default0]: number of documents: 32740750 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.002403 seconds -[default0]: number of documents: 32740750 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_en_train_indexmap_2470768ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_en_train_indexmap_2470768ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.249 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.266264 seconds -[default0]: number of documents: 5413205 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 5413205) total of 5413205 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.002572 seconds -[default0]: number of documents: 5413205 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004093 seconds -[default0]: number of documents: 5413205 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_es_train_indexmap_540571ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_es_train_indexmap_540571ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.534 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.060060 seconds -[default0]: number of documents: 3752156 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 3752156) total of 3752156 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.040171 seconds -[default0]: number of documents: 3752156 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.005179 seconds -[default0]: number of documents: 3752156 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pt_train_indexmap_433493ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pt_train_indexmap_433493ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.208 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.234164 seconds -[default0]: number of documents: 5316403 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 5316403) total of 5316403 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004440 seconds -[default0]: number of documents: 5316403 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.005526 seconds -[default0]: number of documents: 5316403 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fr_train_indexmap_414881ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fr_train_indexmap_414881ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.355 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.253404 seconds -[default0]: number of documents: 2707724 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 2707724) total of 2707724 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.034756 seconds -[default0]: number of documents: 2707724 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.005913 seconds -[default0]: number of documents: 2707724 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_code_train_indexmap_370738ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_code_train_indexmap_370738ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.277 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.195438 seconds -[default0]: number of documents: 2160181 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 2160181) total of 2160181 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.009355 seconds -[default0]: number of documents: 2160181 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004138 seconds -[default0]: number of documents: 2160181 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ar_train_indexmap_294803ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ar_train_indexmap_294803ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.153 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.136364 seconds -[default0]: number of documents: 2643418 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 2643418) total of 2643418 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.005118 seconds -[default0]: number of documents: 2643418 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004112 seconds -[default0]: number of documents: 2643418 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_id_train_indexmap_290321ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_id_train_indexmap_290321ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.134 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.321375 seconds -[default0]: number of documents: 3589234 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 3589234) total of 3589234 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.025577 seconds -[default0]: number of documents: 3589234 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004861 seconds -[default0]: number of documents: 3589234 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zh_train_indexmap_289514ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zh_train_indexmap_289514ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.199 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.088981 seconds -[default0]: number of documents: 1554667 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 1554667) total of 1554667 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.136027 seconds -[default0]: number of documents: 1554667 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004919 seconds -[default0]: number of documents: 1554667 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_hi_train_indexmap_277185ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_hi_train_indexmap_277185ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.114 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.170477 seconds -[default0]: number of documents: 1672106 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 1672106) total of 1672106 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.069778 seconds -[default0]: number of documents: 1672106 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003648 seconds -[default0]: number of documents: 1672106 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_vi_train_indexmap_195289ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_vi_train_indexmap_195289ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.105 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.159032 seconds -[default0]: number of documents: 855756 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 855756) total of 855756 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.055166 seconds -[default0]: number of documents: 855756 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.005066 seconds -[default0]: number of documents: 855756 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ur_train_indexmap_120747ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ur_train_indexmap_120747ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.126 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.041600 seconds -[default0]: number of documents: 584590 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 584590) total of 584590 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.077531 seconds -[default0]: number of documents: 584590 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001915 seconds -[default0]: number of documents: 584590 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_te_train_indexmap_84551ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_te_train_indexmap_84551ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.102 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.054166 seconds -[default0]: number of documents: 415433 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 415433) total of 415433 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.050603 seconds -[default0]: number of documents: 415433 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001649 seconds -[default0]: number of documents: 415433 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ta_train_indexmap_58345ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ta_train_indexmap_58345ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.110 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.075577 seconds -[default0]: number of documents: 428843 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 428843) total of 428843 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.047306 seconds -[default0]: number of documents: 428843 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001685 seconds -[default0]: number of documents: 428843 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bn_train_indexmap_52416ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bn_train_indexmap_52416ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.106 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.069795 seconds -[default0]: number of documents: 417269 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 417269) total of 417269 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.042238 seconds -[default0]: number of documents: 417269 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001799 seconds -[default0]: number of documents: 417269 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_mr_train_indexmap_41937ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_mr_train_indexmap_41937ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.068 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.377393 seconds -[default0]: number of documents: 1130481 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 1130481) total of 1130481 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.185947 seconds -[default0]: number of documents: 1130481 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.008148 seconds -[default0]: number of documents: 1130481 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sw_train_indexmap_35669ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sw_train_indexmap_35669ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.130 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.106292 seconds -[default0]: number of documents: 347499 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 347499) total of 347499 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.045956 seconds -[default0]: number of documents: 347499 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000959 seconds -[default0]: number of documents: 347499 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_gu_train_indexmap_35293ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_gu_train_indexmap_35293ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.048 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.072928 seconds -[default0]: number of documents: 339210 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 339210) total of 339210 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.056132 seconds -[default0]: number of documents: 339210 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001170 seconds -[default0]: number of documents: 339210 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pa_train_indexmap_32937ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pa_train_indexmap_32937ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.065 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.041171 seconds -[default0]: number of documents: 315754 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 315754) total of 315754 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.049549 seconds -[default0]: number of documents: 315754 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001168 seconds -[default0]: number of documents: 315754 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ne_train_indexmap_24781ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ne_train_indexmap_24781ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.053 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.076173 seconds -[default0]: number of documents: 918416 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 918416) total of 918416 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.047878 seconds -[default0]: number of documents: 918416 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.002411 seconds -[default0]: number of documents: 918416 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_yo_train_indexmap_22207ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_yo_train_indexmap_22207ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.059 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.075314 seconds -[default0]: number of documents: 950097 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 950097) total of 950097 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.164501 seconds -[default0]: number of documents: 950097 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.008150 seconds -[default0]: number of documents: 950097 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ig_train_indexmap_20472ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ig_train_indexmap_20472ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.076 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.109307 seconds -[default0]: number of documents: 915063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915063) total of 915063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.045809 seconds -[default0]: number of documents: 915063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.007397 seconds -[default0]: number of documents: 915063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ny_train_indexmap_17130ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ny_train_indexmap_17130ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.095 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.063158 seconds -[default0]: number of documents: 915061 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915061) total of 915061 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.270398 seconds -[default0]: number of documents: 915061 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.008049 seconds -[default0]: number of documents: 915061 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zu_train_indexmap_16600ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zu_train_indexmap_16600ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.074 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.110953 seconds -[default0]: number of documents: 915058 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915058) total of 915058 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.126031 seconds -[default0]: number of documents: 915058 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003401 seconds -[default0]: number of documents: 915058 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_xh_train_indexmap_16031ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_xh_train_indexmap_16031ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.098 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.194360 seconds -[default0]: number of documents: 865056 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 865056) total of 865056 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.029635 seconds -[default0]: number of documents: 865056 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.005401 seconds -[default0]: number of documents: 865056 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sn_train_indexmap_15894ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sn_train_indexmap_15894ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.143 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.089352 seconds -[default0]: number of documents: 915044 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915044) total of 915044 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.075344 seconds -[default0]: number of documents: 915044 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.002254 seconds -[default0]: number of documents: 915044 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ts_train_indexmap_15753ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ts_train_indexmap_15753ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.053 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.071655 seconds -[default0]: number of documents: 915043 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915043) total of 915043 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.218447 seconds -[default0]: number of documents: 915043 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.009749 seconds -[default0]: number of documents: 915043 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rw_train_indexmap_15697ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rw_train_indexmap_15697ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.096 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.117628 seconds -[default0]: number of documents: 915021 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915021) total of 915021 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.089372 seconds -[default0]: number of documents: 915021 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003897 seconds -[default0]: number of documents: 915021 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_lg_train_indexmap_14852ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_lg_train_indexmap_14852ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.067 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.090905 seconds -[default0]: number of documents: 915054 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915054) total of 915054 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.102916 seconds -[default0]: number of documents: 915054 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004062 seconds -[default0]: number of documents: 915054 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tn_train_indexmap_14826ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tn_train_indexmap_14826ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.071 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.066151 seconds -[default0]: number of documents: 915051 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915051) total of 915051 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.369021 seconds -[default0]: number of documents: 915051 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.008515 seconds -[default0]: number of documents: 915051 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_nso_train_indexmap_14460ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_nso_train_indexmap_14460ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.077 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.235961 seconds -[default0]: number of documents: 318189 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 318189) total of 318189 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004257 seconds -[default0]: number of documents: 318189 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001073 seconds -[default0]: number of documents: 318189 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rn_train_indexmap_12148ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rn_train_indexmap_12148ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.051 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.054993 seconds -[default0]: number of documents: 265864 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265864) total of 265864 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.052371 seconds -[default0]: number of documents: 265864 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000950 seconds -[default0]: number of documents: 265864 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ml_train_indexmap_11018ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ml_train_indexmap_11018ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.095 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.024464 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.122191 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001493 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_kn_train_indexmap_10415ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_kn_train_indexmap_10415ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.050 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.039697 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.044422 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001065 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_or_train_indexmap_10164ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_or_train_indexmap_10164ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.049 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.058479 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.048977 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001003 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_as_train_indexmap_9836ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_as_train_indexmap_9836ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.066 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.030351 seconds -[default0]: number of documents: 365060 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 365060) total of 365060 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.062005 seconds -[default0]: number of documents: 365060 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001205 seconds -[default0]: number of documents: 365060 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ln_train_indexmap_7951ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ln_train_indexmap_7951ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.056 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.076966 seconds -[default0]: number of documents: 365063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 365063) total of 365063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.037049 seconds -[default0]: number of documents: 365063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001469 seconds -[default0]: number of documents: 365063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_wo_train_indexmap_7715ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_wo_train_indexmap_7715ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.061 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.042605 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.060363 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001160 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tum_train_indexmap_7304ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tum_train_indexmap_7304ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.047 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.058705 seconds -[default0]: number of documents: 265180 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265180) total of 265180 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.161862 seconds -[default0]: number of documents: 265180 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000797 seconds -[default0]: number of documents: 265180 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ki_train_indexmap_7242ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ki_train_indexmap_7242ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.052 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.035144 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.096689 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001108 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_st_train_indexmap_7181ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_st_train_indexmap_7181ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.051 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.077526 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.079844 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000822 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fon_train_indexmap_7118ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fon_train_indexmap_7118ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.060 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.048945 seconds -[default0]: number of documents: 281199 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 281199) total of 281199 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.081125 seconds -[default0]: number of documents: 281199 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001248 seconds -[default0]: number of documents: 281199 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_eu_train_indexmap_7114ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_eu_train_indexmap_7114ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.116 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.015658 seconds -[default0]: number of documents: 271191 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 271191) total of 271191 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.060124 seconds -[default0]: number of documents: 271191 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001175 seconds -[default0]: number of documents: 271191 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ca_train_indexmap_6963ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ca_train_indexmap_6963ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.053 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.062868 seconds -[default0]: number of documents: 265071 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265071) total of 265071 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.079898 seconds -[default0]: number of documents: 265071 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000784 seconds -[default0]: number of documents: 265071 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ak_train_indexmap_6805ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ak_train_indexmap_6805ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.061 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.097312 seconds -[default0]: number of documents: 265180 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265180) total of 265180 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.076509 seconds -[default0]: number of documents: 265180 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001304 seconds -[default0]: number of documents: 265180 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bm_train_indexmap_6739ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bm_train_indexmap_6739ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.048 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.038738 seconds -[default0]: number of documents: 265071 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265071) total of 265071 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.028788 seconds -[default0]: number of documents: 265071 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000956 seconds -[default0]: number of documents: 265071 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tw_train_indexmap_6691ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tw_train_indexmap_6691ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.038 seconds -[default0]:> building indices for blendable datasets ... -[default0]: > sample ratios: -[default0]: dataset 0, input: 0.387235, achieved: 0.387235 -[default0]: dataset 1, input: 0.0847217, achieved: 0.0847217 -[default0]: dataset 2, input: 0.0679398, achieved: 0.0679398 -[default0]: dataset 3, input: 0.0650228, achieved: 0.0650228 -[default0]: dataset 4, input: 0.0581044, achieved: 0.0581043 -[default0]: dataset 5, input: 0.0462033, achieved: 0.0462033 -[default0]: dataset 6, input: 0.0455009, achieved: 0.0455009 -[default0]: dataset 7, input: 0.0453745, achieved: 0.0453744 -[default0]: dataset 8, input: 0.0434421, achieved: 0.0434421 -[default0]: dataset 9, input: 0.030607, achieved: 0.030607 -[default0]: dataset 10, input: 0.0189242, achieved: 0.0189242 -[default0]: dataset 11, input: 0.0132513, achieved: 0.0132513 -[default0]: dataset 12, input: 0.00914419, achieved: 0.00914417 -[default0]: dataset 13, input: 0.00821496, achieved: 0.00821492 -[default0]: dataset 14, input: 0.0065726, achieved: 0.00657258 -[default0]: dataset 15, input: 0.00559018, achieved: 0.00559023 -[default0]: dataset 16, input: 0.00553131, achieved: 0.00553135 -[default0]: dataset 17, input: 0.00516195, achieved: 0.00516192 -[default0]: dataset 18, input: 0.00388374, achieved: 0.00388376 -[default0]: dataset 19, input: 0.00348029, achieved: 0.00348033 -[default0]: dataset 20, input: 0.00320848, achieved: 0.0032085 -[default0]: dataset 21, input: 0.0026846, achieved: 0.00268464 -[default0]: dataset 22, input: 0.00260158, achieved: 0.00260161 -[default0]: dataset 23, input: 0.00251239, achieved: 0.00251236 -[default0]: dataset 24, input: 0.00249093, achieved: 0.00249096 -[default0]: dataset 25, input: 0.00246883, achieved: 0.00246885 -[default0]: dataset 26, input: 0.00245999, achieved: 0.00245997 -[default0]: dataset 27, input: 0.00232756, achieved: 0.00232756 -[default0]: dataset 28, input: 0.00232361, achieved: 0.00232365 -[default0]: dataset 29, input: 0.00226616, achieved: 0.00226619 -[default0]: dataset 30, input: 0.00190391, achieved: 0.00190387 -[default0]: dataset 31, input: 0.00172681, achieved: 0.0017268 -[default0]: dataset 32, input: 0.00163226, achieved: 0.00163222 -[default0]: dataset 33, input: 0.00159296, achieved: 0.00159297 -[default0]: dataset 34, input: 0.0015415, achieved: 0.00154146 -[default0]: dataset 35, input: 0.00124602, achieved: 0.00124601 -[default0]: dataset 36, input: 0.00120908, achieved: 0.00120907 -[default0]: dataset 37, input: 0.00114468, achieved: 0.00114469 -[default0]: dataset 38, input: 0.00113489, achieved: 0.00113492 -[default0]: dataset 39, input: 0.00112542, achieved: 0.00112542 -[default0]: dataset 40, input: 0.00111548, achieved: 0.00111547 -[default0]: dataset 41, input: 0.00111485, achieved: 0.00111485 -[default0]: dataset 42, input: 0.00109117, achieved: 0.00109114 -[default0]: dataset 43, input: 0.00106639, achieved: 0.00106636 -[default0]: dataset 44, input: 0.00105613, achieved: 0.00105615 -[default0]: dataset 45, input: 0.00104855, achieved: 0.0010486 -[default0]:> elapsed time for building blendable dataset indices: 0.54 (sec) -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.300563 seconds -[default0]: number of documents: 15234080 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [14472376, 15234080) total of 761704 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ar/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ar_text_document_validation_pretraining_indexmap_885ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ar/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ar_text_document_validation_pretraining_indexmap_885ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ar/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ar_text_document_validation_pretraining_indexmap_885ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.160 seconds -[default0]: total number of samples: 221750 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.201631 seconds -[default0]: number of documents: 6142390 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [5835270, 6142390) total of 307120 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ca/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ca_text_document_validation_pretraining_indexmap_301ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ca/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ca_text_document_validation_pretraining_indexmap_301ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ca/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ca_text_document_validation_pretraining_indexmap_301ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.067 seconds -[default0]: total number of samples: 136143 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.189672 seconds -[default0]: number of documents: 26176998 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [24868148, 26176998) total of 1308850 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/code/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_code_text_document_validation_pretraining_indexmap_3486ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/code/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_code_text_document_validation_pretraining_indexmap_3486ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/code/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_code_text_document_validation_pretraining_indexmap_3486ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.132 seconds -[default0]: total number of samples: 432311 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.140548 seconds -[default0]: number of documents: 20844665 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [19802432, 20844665) total of 1042233 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/en/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_en_text_document_validation_pretraining_indexmap_5933ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/en/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_en_text_document_validation_pretraining_indexmap_5933ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/en/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_en_text_document_validation_pretraining_indexmap_5933ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.128 seconds -[default0]: total number of samples: 521545 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.113773 seconds -[default0]: number of documents: 67005817 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [63655526, 67005817) total of 3350291 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/es/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_es_text_document_validation_pretraining_indexmap_2855ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/es/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_es_text_document_validation_pretraining_indexmap_2855ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/es/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_es_text_document_validation_pretraining_indexmap_2855ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.335 seconds -[default0]: total number of samples: 1740321 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.194526 seconds -[default0]: number of documents: 5149795 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [4892305, 5149795) total of 257490 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/eu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_eu_text_document_validation_pretraining_indexmap_42ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/eu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_eu_text_document_validation_pretraining_indexmap_42ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/eu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_eu_text_document_validation_pretraining_indexmap_42ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.048 seconds -[default0]: total number of samples: 26370 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.169211 seconds -[default0]: number of documents: 58847091 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [55904736, 58847091) total of 2942355 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/fr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_fr_text_document_validation_pretraining_indexmap_3493ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/fr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_fr_text_document_validation_pretraining_indexmap_3493ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/fr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_fr_text_document_validation_pretraining_indexmap_3493ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.234 seconds -[default0]: total number of samples: 1458654 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.183237 seconds -[default0]: number of documents: 12514253 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [11888540, 12514253) total of 625713 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/id/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_id_text_document_validation_pretraining_indexmap_293ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/id/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_id_text_document_validation_pretraining_indexmap_293ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/id/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_id_text_document_validation_pretraining_indexmap_293ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.080 seconds -[default0]: total number of samples: 134071 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.066997 seconds -[default0]: number of documents: 180608 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [171578, 180608) total of 9030 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-as/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-as_text_document_validation_pretraining_indexmap_3ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-as/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-as_text_document_validation_pretraining_indexmap_3ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-as/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-as_text_document_validation_pretraining_indexmap_3ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.006 seconds -[default0]: total number of samples: 2501 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.138256 seconds -[default0]: number of documents: 12303134 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [11687977, 12303134) total of 615157 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-bn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-bn_text_document_validation_pretraining_indexmap_147ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-bn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-bn_text_document_validation_pretraining_indexmap_147ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-bn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-bn_text_document_validation_pretraining_indexmap_147ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.080 seconds -[default0]: total number of samples: 157244 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.203146 seconds -[default0]: number of documents: 2033057 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [1931404, 2033057) total of 101653 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-gu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-gu_text_document_validation_pretraining_indexmap_11ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-gu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-gu_text_document_validation_pretraining_indexmap_11ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-gu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-gu_text_document_validation_pretraining_indexmap_11ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.042 seconds -[default0]: total number of samples: 20517 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.303718 seconds -[default0]: number of documents: 26793553 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [25453875, 26793553) total of 1339678 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-hi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-hi_text_document_validation_pretraining_indexmap_200ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-hi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-hi_text_document_validation_pretraining_indexmap_200ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-hi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-hi_text_document_validation_pretraining_indexmap_200ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.101 seconds -[default0]: total number of samples: 101502 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.286973 seconds -[default0]: number of documents: 3155990 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [2998190, 3155990) total of 157800 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-kn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-kn_text_document_validation_pretraining_indexmap_17ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-kn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-kn_text_document_validation_pretraining_indexmap_17ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-kn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-kn_text_document_validation_pretraining_indexmap_17ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.026 seconds -[default0]: total number of samples: 44182 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.221447 seconds -[default0]: number of documents: 6692522 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [6357896, 6692522) total of 334626 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ml/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ml_text_document_validation_pretraining_indexmap_28ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ml/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ml_text_document_validation_pretraining_indexmap_28ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ml/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ml_text_document_validation_pretraining_indexmap_28ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.064 seconds -[default0]: total number of samples: 47613 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.223710 seconds -[default0]: number of documents: 3017261 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [2866398, 3017261) total of 150863 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-mr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-mr_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-mr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-mr_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-mr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-mr_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.040 seconds -[default0]: total number of samples: 29298 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.185810 seconds -[default0]: number of documents: 3648041 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [3465639, 3648041) total of 182402 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ne/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ne_text_document_validation_pretraining_indexmap_18ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ne/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ne_text_document_validation_pretraining_indexmap_18ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ne/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ne_text_document_validation_pretraining_indexmap_18ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.070 seconds -[default0]: total number of samples: 5659 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.152560 seconds -[default0]: number of documents: 4327282 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [4110918, 4327282) total of 216364 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-or/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-or_text_document_validation_pretraining_indexmap_10ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-or/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-or_text_document_validation_pretraining_indexmap_10ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-or/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-or_text_document_validation_pretraining_indexmap_10ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.042 seconds -[default0]: total number of samples: 12423 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.146791 seconds -[default0]: number of documents: 2698896 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [2563951, 2698896) total of 134945 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-pa/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-pa_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-pa/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-pa_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-pa/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-pa_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.030 seconds -[default0]: total number of samples: 19133 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.436784 seconds -[default0]: number of documents: 12767593 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [12129213, 12767593) total of 638380 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ta/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ta_text_document_validation_pretraining_indexmap_57ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ta/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ta_text_document_validation_pretraining_indexmap_57ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ta/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ta_text_document_validation_pretraining_indexmap_57ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.069 seconds -[default0]: total number of samples: 87928 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.141326 seconds -[default0]: number of documents: 4342323 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [4125207, 4342323) total of 217116 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-te/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-te_text_document_validation_pretraining_indexmap_25ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-te/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-te_text_document_validation_pretraining_indexmap_25ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-te/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-te_text_document_validation_pretraining_indexmap_25ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.054 seconds -[default0]: total number of samples: 69780 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.410133 seconds -[default0]: number of documents: 3022722 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [2871586, 3022722) total of 151136 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ur/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ur_text_document_validation_pretraining_indexmap_34ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ur/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ur_text_document_validation_pretraining_indexmap_34ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ur/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ur_text_document_validation_pretraining_indexmap_34ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.011 seconds -[default0]: total number of samples: 22532 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.145349 seconds -[default0]: number of documents: 1162568 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [1104440, 1162568) total of 58128 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/nigercongo-all/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_nigercongo-all_text_document_validation_pretraining_indexmap_9ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/nigercongo-all/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_nigercongo-all_text_document_validation_pretraining_indexmap_9ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/nigercongo-all/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_nigercongo-all_text_document_validation_pretraining_indexmap_9ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.008 seconds -[default0]: total number of samples: 1608 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.177521 seconds -[default0]: number of documents: 55294645 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [52529913, 55294645) total of 2764732 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-en/meg_ds_bigscience_tokenizer_text_document_validation_pretraining_indexmap_2178ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-en/meg_ds_bigscience_tokenizer_text_document_validation_pretraining_indexmap_2178ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-en/meg_ds_bigscience_tokenizer_text_document_validation_pretraining_indexmap_2178ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.149 seconds -[default0]: total number of samples: 690621 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.151570 seconds -[default0]: number of documents: 44855616 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [42612835, 44855616) total of 2242781 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-zh/meg_ds_bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_text_document_validation_pretraining_indexmap_1480ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-zh/meg_ds_bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_text_document_validation_pretraining_indexmap_1480ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-zh/meg_ds_bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_text_document_validation_pretraining_indexmap_1480ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.121 seconds -[default0]: total number of samples: 468689 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.577527 seconds -[default0]: number of documents: 31969891 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [30371396, 31969891) total of 1598495 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/pt/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_pt_text_document_validation_pretraining_indexmap_1326ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/pt/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_pt_text_document_validation_pretraining_indexmap_1326ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/pt/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_pt_text_document_validation_pretraining_indexmap_1326ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.150 seconds -[default0]: total number of samples: 497625 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.225993 seconds -[default0]: number of documents: 34110375 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [32404856, 34110375) total of 1705519 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/vi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_vi_text_document_validation_pretraining_indexmap_659ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/vi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_vi_text_document_validation_pretraining_indexmap_659ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/vi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_vi_text_document_validation_pretraining_indexmap_659ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.126 seconds -[default0]: total number of samples: 125120 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.076359 seconds -[default0]: number of documents: 43761623 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [41573542, 43761623) total of 2188081 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zhs/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zhs_text_document_validation_pretraining_indexmap_3236ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zhs/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zhs_text_document_validation_pretraining_indexmap_3236ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zhs/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zhs_text_document_validation_pretraining_indexmap_3236ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.243 seconds -[default0]: total number of samples: 1010592 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.161097 seconds -[default0]: number of documents: 197602 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [187722, 197602) total of 9880 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zht/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zht_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zht/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zht_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zht/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zht_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.019 seconds -[default0]: total number of samples: 4451 -[default0]: total number of epochs: 1 -[default0]:> building indices for blendable datasets ... -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default0]: > sample ratios: -[default0]: dataset 0, input: 0.0330676, achieved: 0.0330676 -[default0]: dataset 1, input: 0.0112421, achieved: 0.0112421 -[default0]: dataset 2, input: 0.130272, achieved: 0.130272 -[default0]: dataset 3, input: 0.221712, achieved: 0.221712 -[default0]: dataset 4, input: 0.106678, achieved: 0.106678 -[default0]: dataset 5, input: 0.00155951, achieved: 0.00155955 -[default0]: dataset 6, input: 0.13054, achieved: 0.13054 -[default0]: dataset 7, input: 0.010918, achieved: 0.0109181 -[default0]: dataset 8, input: 0.000110214, achieved: 0.000110257 -[default0]: dataset 9, input: 0.00549238, achieved: 0.00549235 -[default0]: dataset 10, input: 0.000402122, achieved: 0.000402094 -[default0]: dataset 11, input: 0.00747007, achieved: 0.00747007 -[default0]: dataset 12, input: 0.000619047, achieved: 0.000619024 -[default0]: dataset 13, input: 0.00103353, achieved: 0.0010336 -[default0]: dataset 14, input: 0.000501201, achieved: 0.000501226 -[default0]: dataset 15, input: 0.000667277, achieved: 0.000667231 -[default0]: dataset 16, input: 0.000359281, achieved: 0.000359326 -[default0]: dataset 17, input: 0.000508443, achieved: 0.000508519 -[default0]: dataset 18, input: 0.00211373, achieved: 0.0021138 -[default0]: dataset 19, input: 0.000912995, achieved: 0.000912961 -[default0]: dataset 20, input: 0.00124543, achieved: 0.00124546 -[default0]: dataset 21, input: 0.000315887, achieved: 0.00031594 -[default0]: dataset 22, input: 0.0813721, achieved: 0.0813721 -[default0]: dataset 23, input: 0.0552939, achieved: 0.0552939 -[default0]: dataset 24, input: 0.0495415, achieved: 0.0495414 -[default0]: dataset 25, input: 0.0246164, achieved: 0.0246163 -[default0]: dataset 26, input: 0.120917, achieved: 0.120917 -[default0]: dataset 27, input: 0.000517703, achieved: 0.000517666 -[default0]:> elapsed time for building blendable dataset indices: 0.32 (sec) -[default0]:> finished creating T0 datasets ... -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default6]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default2]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default3]:GOTCONSUMEDSAMPLES 0 0 -[default7]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default5]:GOTCONSUMEDSAMPLES 0 0 -[default1]:GOTCONSUMEDSAMPLES 0 0 -[default4]:GOTCONSUMEDSAMPLES 0 0 -[default0]:GOTCONSUMEDSAMPLES 0 0 -[default0]:[after dataloaders are built] datetime: 2022-09-07 21:54:45 -[default0]:done with setup ... -[default0]:training ... -[default0]:Number of parameters: [tensor rank - pipeline rank] w/ and w/o embeddings: -[default0]:[000-000] 258.9584B / 0.0021B -[default4]:[000-057] 177.5835B / 177.5835B -[default0]:[000-010] 177.5835B / 177.5835B -[default0]:[000-070] 177.5855B / 177.5855B -[default7]:time (ms) | model-and-optimizer-setup: 30508.72 | train/valid/test-data-iterators-setup: 35925.60 -[default0]:[000-006] 177.5835B / 177.5835B -[default4]:[000-063] 177.5835B / 177.5835B -[default4]:[000-043] 177.5835B / 177.5835B -[default0]:[000-066] 177.5835B / 177.5835B -[default0]:[000-004] 177.5835B / 177.5835B -[default0]:[000-062] 177.5835B / 177.5835B -[default4]:[000-007] 177.5835B / 177.5835B -[default4]:[000-059] 177.5835B / 177.5835B -[default4]:[000-037] 177.5835B / 177.5835B -[default0]:[000-026] 177.5835B / 177.5835B -[default0]:[000-036] 177.5835B / 177.5835B -[default0]:[000-068] 177.5835B / 177.5835B -[default4]:[000-009] 177.5835B / 177.5835B -[default0]:[000-048] 177.5835B / 177.5835B -[default0]:[000-058] 177.5835B / 177.5835B -[default0]:[000-024] 177.5835B / 177.5835B -[default0]:[000-044] 177.5835B / 177.5835B -[default0]:[000-032] 177.5835B / 177.5835B -[default0]:[000-046] 177.5835B / 177.5835B -[default4]:[000-011] 177.5835B / 177.5835B -[default0]:[000-030] 177.5835B / 177.5835B -[default4]:[000-045] 177.5835B / 177.5835B -[default0]:[000-052] 177.5835B / 177.5835B -[default0]:[000-028] 177.5835B / 177.5835B -[default0]:[000-014] 177.5835B / 177.5835B -[default4]:[000-033] 177.5835B / 177.5835B -[default0]:[000-050] 177.5835B / 177.5835B -[default0]:[000-020] 177.5835B / 177.5835B -[default4]:[000-031] 177.5835B / 177.5835B -[default0]:[000-042] 177.5835B / 177.5835B -[default4]:[000-069] 177.5835B / 177.5835B -[default4]:[000-021] 177.5835B / 177.5835B -[default4]:[000-071] 258.9563B / 0.0000B -[default4]:[000-029] 177.5835B / 177.5835B -[default0]:[000-008] 177.5835B / 177.5835B -[default0]:[000-016] 177.5835B / 177.5835B -[default4]:[000-015] 177.5835B / 177.5835B -[default0]:[000-022] 177.5835B / 177.5835B -[default0]:[000-012] 177.5835B / 177.5835B -[default0]:[000-034] 177.5835B / 177.5835B -[default4]:[000-065] 177.5835B / 177.5835B -[default0]:[000-060] 177.5835B / 177.5835B -[default4]:[000-039] 177.5835B / 177.5835B -[default0]:[000-038] 177.5835B / 177.5835B -[default4]:[000-003] 177.5835B / 177.5835B -[default4]:[000-047] 177.5835B / 177.5835B -[default4]:[000-023] 177.5835B / 177.5835B -[default4]:[000-017] 177.5835B / 177.5835B -[default4]:[000-067] 177.5835B / 177.5835B -[default4]:[000-013] 177.5835B / 177.5835B -[default4]:[000-041] 177.5835B / 177.5835B -[default0]:[000-056] 177.5835B / 177.5835B -[default4]:[000-049] 177.5835B / 177.5835B -[default4]:[000-051] 177.5835B / 177.5835B -[default0]:[000-002] 177.5835B / 177.5835B -[default0]:[000-018] 177.5835B / 177.5835B -[default4]:[000-035] 177.5835B / 177.5835B -[default4]:[000-055] 177.5835B / 177.5835B -[default4]:[000-053] 177.5835B / 177.5835B -[default0]:[000-054] 177.5835B / 177.5835B -[default4]:[000-025] 177.5835B / 177.5835B -[default4]:[000-019] 177.5835B / 177.5835B -[default4]:[000-027] 177.5835B / 177.5835B -[default0]:[000-064] 177.5835B / 177.5835B -[default4]:[000-061] 177.5835B / 177.5835B -[default4]:[000-005] 177.5835B / 177.5835B -[default4]:[000-001] 177.5835B / 177.5835B -[default0]:[000-040] 177.5835B / 177.5835B -[default0]:[before the start of training step] datetime: 2022-09-07 21:54:45 -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default4]:[Rank 20] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 41086.39501953125 | reserved: 46486.0 | max reserved: 46486.0 -[default4]:[Rank 4] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 41790.39501953125 | reserved: 47382.0 | max reserved: 47382.0 -[default4]:[Rank 100] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37566.39501953125 | reserved: 42902.0 | max reserved: 42902.0 -[default0]:[Rank 0] (after 1 iterations) memory (MB) | allocated: 38080.58544921875 | max allocated: 62086.80322265625 | reserved: 76022.0 | max reserved: 76022.0 -[default0]:[Rank 40] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40206.39501953125 | reserved: 45590.0 | max reserved: 45590.0 -[default0]:[Rank 160] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34926.39501953125 | reserved: 40214.0 | max reserved: 40214.0 -[default4]:[Rank 228] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31934.39501953125 | reserved: 37526.0 | max reserved: 37526.0 -[default0]:[Rank 280] (after 1 iterations) memory (MB) | allocated: 25990.69677734375 | max allocated: 29702.71142578125 | reserved: 34838.0 | max reserved: 34838.0 -[default0]:[Rank 104] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37390.39501953125 | reserved: 44078.0 | max reserved: 44078.0 -[default4]:[Rank 236] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31582.39501953125 | reserved: 42118.0 | max reserved: 42118.0 -[default7]: iteration 1/ 3100 | consumed samples: 2048 | consumed tokens: 4194304 | elapsed time per iteration (s): 218.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 3.396775E+00 | grad norm: 22.263 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 9.384 | TFLOPs: 95.79 | -[default4]:[Rank 252] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 30878.39501953125 | reserved: 36910.0 | max reserved: 36910.0 -[default0]:[Rank 248] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31054.39501953125 | reserved: 36630.0 | max reserved: 36630.0 -[default0]:[Rank 192] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 33518.39501953125 | reserved: 39598.0 | max reserved: 39598.0 -[default0]:[Rank 264] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 30350.39501953125 | reserved: 35734.0 | max reserved: 35734.0 -[default0]:[Rank 16] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 41262.39501953125 | reserved: 46486.0 | max reserved: 46486.0 -[default0]:[Rank 24] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40910.39501953125 | reserved: 46486.0 | max reserved: 46486.0 -[default4]:[Rank 28] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40734.39501953125 | reserved: 46766.0 | max reserved: 46766.0 -[default4]:[Rank 148] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35454.39501953125 | reserved: 42286.0 | max reserved: 42286.0 -[default4]:[Rank 172] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34398.39501953125 | reserved: 40494.0 | max reserved: 40494.0 -[default0]:[Rank 184] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 33870.39501953125 | reserved: 39318.0 | max reserved: 39318.0 -[default0]:[Rank 144] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35630.39501953125 | reserved: 41110.0 | max reserved: 41110.0 -[default0]:[Rank 272] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 29998.39501953125 | reserved: 36014.0 | max reserved: 36014.0 -[default0]:[Rank 96] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37742.39501953125 | reserved: 42902.0 | max reserved: 42902.0 -[default0]:[Rank 232] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31758.39501953125 | reserved: 37806.0 | max reserved: 37806.0 -[default0]:[Rank 120] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 36686.39501953125 | reserved: 42006.0 | max reserved: 42006.0 -[default4]:[Rank 36] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40382.39501953125 | reserved: 45590.0 | max reserved: 45590.0 -[default0]:[Rank 80] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38446.39501953125 | reserved: 43798.0 | max reserved: 43798.0 -[default4]:[Rank 44] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40030.39501953125 | reserved: 45590.0 | max reserved: 45590.0 -[default0]:[Rank 128] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 36334.39501953125 | reserved: 43182.0 | max reserved: 43182.0 -[default4]:[Rank 132] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 36158.39501953125 | reserved: 41110.0 | max reserved: 41110.0 -[default4]:[Rank 68] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38974.39501953125 | reserved: 44974.0 | max reserved: 44974.0 -[default4]:[Rank 84] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38270.39501953125 | reserved: 43798.0 | max reserved: 43798.0 -[default4]:[Rank 124] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 36510.39501953125 | reserved: 42006.0 | max reserved: 42006.0 -[default0]:[Rank 88] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38094.39501953125 | reserved: 44078.0 | max reserved: 44078.0 -[default0]:[Rank 136] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35982.39501953125 | reserved: 41222.0 | max reserved: 41222.0 -[default4]:[Rank 92] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37918.39501953125 | reserved: 44078.0 | max reserved: 44078.0 -[default0]:[Rank 32] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40558.39501953125 | reserved: 45590.0 | max reserved: 45590.0 -[default0]:[Rank 112] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37038.39501953125 | reserved: 42006.0 | max reserved: 42006.0 -[default0]:[Rank 200] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 33166.39501953125 | reserved: 38422.0 | max reserved: 38422.0 -[default0]:[Rank 208] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32814.39501953125 | reserved: 38422.0 | max reserved: 38422.0 -[default4]:[Rank 180] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34046.39501953125 | reserved: 39318.0 | max reserved: 39318.0 -[default0]:[Rank 56] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 39502.39501953125 | reserved: 44694.0 | max reserved: 44694.0 -[default0]:[Rank 176] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34222.39501953125 | reserved: 39430.0 | max reserved: 39430.0 -[default0]:[Rank 48] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 39854.39501953125 | reserved: 45870.0 | max reserved: 45870.0 -[default4]:[Rank 284] (after 1 iterations) memory (MB) | allocated: 41930.33251953125 | max allocated: 55650.33203125 | reserved: 68848.0 | max reserved: 68848.0 -[default4]:[Rank 116] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 36862.39501953125 | reserved: 42006.0 | max reserved: 42006.0 -[default4]:[Rank 276] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 29822.39501953125 | reserved: 34838.0 | max reserved: 34838.0 -[default0]:[Rank 168] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34574.39501953125 | reserved: 41390.0 | max reserved: 41390.0 -[default4]:[Rank 140] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35806.39501953125 | reserved: 41110.0 | max reserved: 41110.0 -[default4]:[Rank 12] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 41438.39501953125 | reserved: 46486.0 | max reserved: 46486.0 -[default0]:[Rank 240] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31406.39501953125 | reserved: 36630.0 | max reserved: 36630.0 -[default4]:[Rank 260] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 30526.39501953125 | reserved: 35734.0 | max reserved: 35734.0 -[default4]:[Rank 60] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 39326.39501953125 | reserved: 44694.0 | max reserved: 44694.0 -[default4]:[Rank 156] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35102.39501953125 | reserved: 40326.0 | max reserved: 40326.0 -[default0]:[Rank 8] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 41614.39501953125 | reserved: 47662.0 | max reserved: 47662.0 -[default4]:[Rank 52] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 39678.39501953125 | reserved: 44694.0 | max reserved: 44694.0 -[default0]:[Rank 72] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38798.39501953125 | reserved: 43798.0 | max reserved: 43798.0 -[default0]:[Rank 256] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 30702.39501953125 | reserved: 36910.0 | max reserved: 36910.0 -[default4]:[Rank 196] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 33342.39501953125 | reserved: 38422.0 | max reserved: 38422.0 -[default4]:[Rank 268] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 30174.39501953125 | reserved: 35734.0 | max reserved: 35734.0 -[default4]:[Rank 188] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 33694.39501953125 | reserved: 40494.0 | max reserved: 40494.0 -[default0]:[Rank 64] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 39150.39501953125 | reserved: 44694.0 | max reserved: 44694.0 -[default0]:[Rank 152] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35278.39501953125 | reserved: 40214.0 | max reserved: 40214.0 -[default4]:[Rank 164] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34750.39501953125 | reserved: 40214.0 | max reserved: 40214.0 -[default4]:[Rank 244] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31230.39501953125 | reserved: 36630.0 | max reserved: 36630.0 -[default0]:[Rank 224] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32110.39501953125 | reserved: 37526.0 | max reserved: 37526.0 -[default4]:[Rank 220] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32286.39501953125 | reserved: 37526.0 | max reserved: 37526.0 -[default4]:[Rank 108] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37214.39501953125 | reserved: 44078.0 | max reserved: 44078.0 -[default4]:[Rank 204] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32990.39501953125 | reserved: 38422.0 | max reserved: 38422.0 -[default4]:[Rank 76] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38622.39501953125 | reserved: 43798.0 | max reserved: 43798.0 -[default4]:[Rank 212] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32638.39501953125 | reserved: 38702.0 | max reserved: 38702.0 -[default0]:[Rank 216] (after 1 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32462.39501953125 | reserved: 37526.0 | max reserved: 37526.0 -[default7]: iteration 2/ 3100 | consumed samples: 4096 | consumed tokens: 8388608 | elapsed time per iteration (s): 143.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 2.484104E+00 | grad norm: 21.382 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.230 | TFLOPs: 145.27 | -[default7]: iteration 3/ 3100 | consumed samples: 6144 | consumed tokens: 12582912 | elapsed time per iteration (s): 141.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 2.453157E+00 | grad norm: 17.740 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.487 | TFLOPs: 147.89 | -[default7]: iteration 4/ 3100 | consumed samples: 8192 | consumed tokens: 16777216 | elapsed time per iteration (s): 144.20 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 2.092513E+00 | grad norm: 5.242 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.203 | TFLOPs: 144.99 | -[default7]: iteration 5/ 3100 | consumed samples: 10240 | consumed tokens: 20971520 | elapsed time per iteration (s): 141.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.979219E+00 | grad norm: 4.324 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.525 | TFLOPs: 148.27 | -[default4]:[2022-09-07 22:07:54,349] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,318] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,349] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt... -[default0]:saving checkpoint at iteration 5 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-07 22:07:54,301] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step5 is begin to save! -[default4]:[2022-09-07 22:07:54,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,396] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,401] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,426] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,426] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,395] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,466] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,401] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,464] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,466] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,464] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,466] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,497] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,497] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,497] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,496] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,517] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,497] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,466] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,466] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,517] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,466] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,563] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,496] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,563] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,498] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,624] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,600] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,618] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_71_model_states.pt... -[default4]:[2022-09-07 22:07:54,656] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_71_model_states.pt. -[default4]:[2022-09-07 22:07:54,646] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,652] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,646] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,624] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,600] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,600] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,654] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,646] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,600] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,654] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,624] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,654] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,624] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,624] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,624] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,624] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,646] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,645] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:54,645] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt... -[default4]:[2022-09-07 22:07:54,652] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:57,630] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:57,631] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_26_model_states.pt... -[default0]:[2022-09-07 22:07:57,637] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_26_model_states.pt. -[default0]:[2022-09-07 22:07:57,703] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:57,703] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_08_model_states.pt... -[default0]:[2022-09-07 22:07:57,707] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_08_model_states.pt. -[default0]:[2022-09-07 22:07:57,759] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:57,759] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_22_model_states.pt... -[default0]:[2022-09-07 22:07:57,763] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_22_model_states.pt. -[default0]:[2022-09-07 22:07:57,753] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:57,754] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_10_model_states.pt... -[default0]:[2022-09-07 22:07:57,758] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_10_model_states.pt. -[default0]:[2022-09-07 22:07:57,845] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:57,845] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_12_model_states.pt... -[default0]:[2022-09-07 22:07:57,852] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_12_model_states.pt. -[default4]:[2022-09-07 22:07:57,951] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:57,951] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_33_model_states.pt... -[default4]:[2022-09-07 22:07:57,963] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:57,963] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_29_model_states.pt... -[default0]:[2022-09-07 22:07:57,961] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:57,962] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_16_model_states.pt... -[default0]:[2022-09-07 22:07:57,990] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_16_model_states.pt. -[default0]:[2022-09-07 22:07:57,961] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:57,961] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_62_model_states.pt... -[default0]:[2022-09-07 22:07:57,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_62_model_states.pt. -[default0]:[2022-09-07 22:07:57,995] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:57,995] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_66_model_states.pt... -[default0]:[2022-09-07 22:07:57,999] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_66_model_states.pt. -[default0]:[2022-09-07 22:07:58,041] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,041] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_68_model_states.pt... -[default0]:[2022-09-07 22:07:58,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_68_model_states.pt. -[default4]:[2022-09-07 22:07:57,963] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_33_model_states.pt. -[default4]:[2022-09-07 22:07:58,060] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,060] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_15_model_states.pt... -[default0]:[2022-09-07 22:07:58,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,051] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_42_model_states.pt... -[default4]:[2022-09-07 22:07:58,019] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,019] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_23_model_states.pt... -[default4]:[2022-09-07 22:07:58,022] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_23_model_states.pt. -[default0]:[2022-09-07 22:07:58,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,021] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_28_model_states.pt... -[default0]:[2022-09-07 22:07:58,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_28_model_states.pt. -[default4]:[2022-09-07 22:07:57,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_29_model_states.pt. -[default4]:[2022-09-07 22:07:58,041] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,041] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_09_model_states.pt... -[default4]:[2022-09-07 22:07:58,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_09_model_states.pt. -[default4]:[2022-09-07 22:07:58,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,051] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_43_model_states.pt... -[default4]:[2022-09-07 22:07:58,067] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_43_model_states.pt. -[default0]:[2022-09-07 22:07:58,107] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,107] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_24_model_states.pt... -[default0]:[2022-09-07 22:07:58,132] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_24_model_states.pt. -[default4]:[2022-09-07 22:07:58,080] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,080] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_11_model_states.pt... -[default4]:[2022-09-07 22:07:58,083] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_11_model_states.pt. -[default4]:[2022-09-07 22:07:58,146] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,147] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_13_model_states.pt... -[default4]:[2022-09-07 22:07:58,150] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_13_model_states.pt. -[default0]:[2022-09-07 22:07:58,122] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,122] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_14_model_states.pt... -[default0]:[2022-09-07 22:07:58,124] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_14_model_states.pt. -[default4]:[2022-09-07 22:07:58,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_15_model_states.pt. -[default0]:[2022-09-07 22:07:58,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_42_model_states.pt. -[default4]:[2022-09-07 22:07:58,107] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,107] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_69_model_states.pt... -[default4]:[2022-09-07 22:07:58,121] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_69_model_states.pt. -[default0]:[2022-09-07 22:07:58,166] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,166] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_58_model_states.pt... -[default0]:[2022-09-07 22:07:58,094] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,094] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_32_model_states.pt... -[default0]:[2022-09-07 22:07:58,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_32_model_states.pt. -[default4]:[2022-09-07 22:07:58,108] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,108] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_27_model_states.pt... -[default4]:[2022-09-07 22:07:58,122] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_27_model_states.pt. -[default0]:[2022-09-07 22:07:58,179] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,179] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_20_model_states.pt... -[default0]:[2022-09-07 22:07:58,194] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_20_model_states.pt. -[default4]:[2022-09-07 22:07:58,191] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,191] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_31_model_states.pt... -[default4]:[2022-09-07 22:07:58,204] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_31_model_states.pt. -[default4]:[2022-09-07 22:07:58,249] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,250] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_03_model_states.pt... -[default4]:[2022-09-07 22:07:58,245] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,245] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_21_model_states.pt... -[default4]:[2022-09-07 22:07:58,248] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_21_model_states.pt. -[default4]:[2022-09-07 22:07:58,180] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,181] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_17_model_states.pt... -[default4]:[2022-09-07 22:07:58,195] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_17_model_states.pt. -[default0]:[2022-09-07 22:07:58,240] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,241] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_50_model_states.pt... -[default0]:[2022-09-07 22:07:58,245] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_50_model_states.pt. -[default0]:[2022-09-07 22:07:58,246] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,247] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_52_model_states.pt... -[default0]:[2022-09-07 22:07:58,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_52_model_states.pt. -[default4]:[2022-09-07 22:07:58,185] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_25_model_states.pt... -[default4]:[2022-09-07 22:07:58,194] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_25_model_states.pt. -[default0]:[2022-09-07 22:07:58,178] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_58_model_states.pt. -[default4]:[2022-09-07 22:07:58,298] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,298] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_53_model_states.pt... -[default0]:[2022-09-07 22:07:58,225] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,226] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_04_model_states.pt... -[default0]:[2022-09-07 22:07:58,231] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_04_model_states.pt. -[default4]:[2022-09-07 22:07:58,293] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_07_model_states.pt... -[default4]:[2022-09-07 22:07:58,298] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_07_model_states.pt. -[default0]:[2022-09-07 22:07:58,278] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,279] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_30_model_states.pt... -[default0]:[2022-09-07 22:07:58,283] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_30_model_states.pt. -[default0]:[2022-09-07 22:07:58,342] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,343] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_06_model_states.pt... -[default0]:[2022-09-07 22:07:58,345] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_06_model_states.pt. -[default4]:[2022-09-07 22:07:58,260] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_03_model_states.pt. -[default0]:[2022-09-07 22:07:58,259] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,259] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_34_model_states.pt... -[default0]:[2022-09-07 22:07:58,270] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_34_model_states.pt. -[default0]:[2022-09-07 22:07:58,288] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,288] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_18_model_states.pt... -[default0]:[2022-09-07 22:07:58,291] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_18_model_states.pt. -[default0]:[2022-09-07 22:07:58,303] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,303] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_02_model_states.pt... -[default0]:[2022-09-07 22:07:58,306] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_02_model_states.pt. -[default4]:[2022-09-07 22:07:58,332] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,333] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_67_model_states.pt... -[default4]:[2022-09-07 22:07:58,336] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_67_model_states.pt. -[default4]:[2022-09-07 22:07:58,327] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,327] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_41_model_states.pt... -[default4]:[2022-09-07 22:07:58,330] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_41_model_states.pt. -[default4]:[2022-09-07 22:07:58,348] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,349] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_19_model_states.pt... -[default4]:[2022-09-07 22:07:58,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_19_model_states.pt. -[default4]:[2022-09-07 22:07:58,320] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,321] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_51_model_states.pt... -[default4]:[2022-09-07 22:07:58,323] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_51_model_states.pt. -[default0]:[2022-09-07 22:07:58,393] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,393] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_40_model_states.pt... -[default4]:[2022-09-07 22:07:58,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,336] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_05_model_states.pt... -[default4]:[2022-09-07 22:07:58,338] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_05_model_states.pt. -[default4]:[2022-09-07 22:07:58,301] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_53_model_states.pt. -[default4]:[2022-09-07 22:07:58,384] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,384] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_59_model_states.pt... -[default4]:[2022-09-07 22:07:58,392] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_59_model_states.pt. -[default0]:[2022-09-07 22:07:58,424] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,425] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_48_model_states.pt... -[default0]:[2022-09-07 22:07:58,431] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_48_model_states.pt. -[default4]:[2022-09-07 22:07:58,383] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,383] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_35_model_states.pt... -[default4]:[2022-09-07 22:07:58,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_35_model_states.pt. -[default4]:[2022-09-07 22:07:58,445] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,445] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_39_model_states.pt... -[default4]:[2022-09-07 22:07:58,450] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_39_model_states.pt. -[default0]:[2022-09-07 22:07:58,473] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,395] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_40_model_states.pt. -[default0]:[2022-09-07 22:07:58,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,416] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_56_model_states.pt... -[default0]:[2022-09-07 22:07:58,419] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_56_model_states.pt. -[default4]:[2022-09-07 22:07:58,483] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,483] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_63_model_states.pt... -[default4]:[2022-09-07 22:07:58,487] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_63_model_states.pt. -[default4]:[2022-09-07 22:07:58,498] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,499] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_49_model_states.pt... -[default4]:[2022-09-07 22:07:58,501] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_49_model_states.pt. -[default0]:[2022-09-07 22:07:58,473] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_38_model_states.pt... -[default0]:[2022-09-07 22:07:58,475] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_38_model_states.pt. -[default0]:[2022-09-07 22:07:58,517] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,518] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_64_model_states.pt... -[default0]:[2022-09-07 22:07:58,520] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_64_model_states.pt. -[default4]:[2022-09-07 22:07:58,509] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,509] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_57_model_states.pt... -[default4]:[2022-09-07 22:07:58,511] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_57_model_states.pt. -[default0]:[2022-09-07 22:07:58,498] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,498] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_44_model_states.pt... -[default0]:[2022-09-07 22:07:58,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_44_model_states.pt. -[default4]:[2022-09-07 22:07:58,554] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,555] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_65_model_states.pt... -[default4]:[2022-09-07 22:07:58,557] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_65_model_states.pt. -[default4]:[2022-09-07 22:07:58,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,562] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_55_model_states.pt... -[default4]:[2022-09-07 22:07:58,566] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_55_model_states.pt. -[default4]:[2022-09-07 22:07:58,628] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,628] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_37_model_states.pt... -[default4]:[2022-09-07 22:07:58,631] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_37_model_states.pt. -[default0]:[2022-09-07 22:07:58,635] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,636] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_60_model_states.pt... -[default0]:[2022-09-07 22:07:58,641] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_60_model_states.pt. -[default4]:[2022-09-07 22:07:58,582] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,582] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_45_model_states.pt... -[default4]:[2022-09-07 22:07:58,584] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_45_model_states.pt. -[default4]:[2022-09-07 22:07:58,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_47_model_states.pt... -[default4]:[2022-09-07 22:07:58,659] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt. -[default4]:[2022-09-07 22:07:58,660] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_61_model_states.pt... -[default4]:[2022-09-07 22:07:58,662] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_61_model_states.pt. -[default0]:[2022-09-07 22:07:58,698] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,698] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt... -[default0]:[2022-09-07 22:07:58,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,700] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_70_model_states.pt... -[default0]:[2022-09-07 22:07:58,702] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_70_model_states.pt. -[default0]:[2022-09-07 22:07:58,652] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,653] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_54_model_states.pt... -[default0]:[2022-09-07 22:07:58,654] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_54_model_states.pt. -[default0]:[2022-09-07 22:07:58,701] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,701] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_46_model_states.pt... -[default0]:[2022-09-07 22:07:58,704] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_46_model_states.pt. -[default0]:[2022-09-07 22:07:58,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt. -[default0]:[2022-09-07 22:07:58,672] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_36_model_states.pt... -[default0]:[2022-09-07 22:07:58,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_36_model_states.pt. -[default4]:[2022-09-07 22:07:58,674] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_47_model_states.pt. -[default4]:[2022-09-07 22:08:00,773] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt. -[default4]:[2022-09-07 22:08:00,774] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_01_model_states.pt... -[default0]:[2022-09-07 22:08:00,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt. -[default0]:[2022-09-07 22:08:00,784] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt -[default0]:[2022-09-07 22:08:00,784] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:08:00,786] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:08:00,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_01_model_states.pt. -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default1]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default6]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default7]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default7]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default1]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default7]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default6]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default2]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default6]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default1]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default5]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default7]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default0]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default5]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default2]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default2]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default4]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default6]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default6]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default3]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default2]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default6]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default3]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default6]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default4]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default5]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default0]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default4]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default3]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default3]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default0]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default1]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default5]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default4]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default5]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default0]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default3]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default4]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default3]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default0]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default7]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default7]:[2022-09-07 22:08:00,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default2]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default2]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default2]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default1]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default2]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default0]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default5]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default6]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default4]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default3]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default0]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default7]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default4]:[2022-09-07 22:08:00,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default5]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default1]:[2022-09-07 22:08:00,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default2]:[2022-09-07 22:08:08,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-07 22:08:08,579] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default0]:[2022-09-07 22:08:08,725] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-07 22:08:08,725] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default0]:[2022-09-07 22:08:08,758] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-07 22:08:08,759] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default3]:[2022-09-07 22:08:08,937] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-07 22:08:08,937] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default7]:[2022-09-07 22:08:09,055] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-07 22:08:09,055] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default0]:[2022-09-07 22:08:09,105] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-07 22:08:09,105] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default0]:[2022-09-07 22:08:09,211] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-07 22:08:09,211] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default1]:[2022-09-07 22:08:09,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-07 22:08:09,336] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default5]:[2022-09-07 22:08:09,408] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-07 22:08:09,408] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default1]:[2022-09-07 22:08:09,466] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-07 22:08:09,467] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default0]:[2022-09-07 22:08:09,714] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-07 22:08:09,714] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default7]:[2022-09-07 22:08:09,796] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-07 22:08:09,796] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default1]:[2022-09-07 22:08:09,768] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-07 22:08:09,769] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default6]:[2022-09-07 22:08:09,873] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-07 22:08:09,873] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default4]:[2022-09-07 22:08:09,968] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-07 22:08:09,968] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default3]:[2022-09-07 22:08:09,983] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-07 22:08:09,983] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default5]:[2022-09-07 22:08:10,026] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-07 22:08:10,026] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default1]:[2022-09-07 22:08:10,162] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-07 22:08:10,162] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default3]:[2022-09-07 22:08:10,138] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-07 22:08:10,138] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default0]:[2022-09-07 22:08:10,163] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-07 22:08:10,163] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default4]:[2022-09-07 22:08:10,238] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-07 22:08:10,239] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default2]:[2022-09-07 22:08:10,309] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-07 22:08:10,309] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default0]:[2022-09-07 22:08:10,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-07 22:08:10,319] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default4]:[2022-09-07 22:08:10,328] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-07 22:08:10,328] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default5]:[2022-09-07 22:08:10,374] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-07 22:08:10,374] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default1]:[2022-09-07 22:08:10,443] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-07 22:08:10,443] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default6]:[2022-09-07 22:08:10,588] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-07 22:08:10,588] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default6]:[2022-09-07 22:08:10,629] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-07 22:08:10,629] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default4]:[2022-09-07 22:08:10,612] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-07 22:08:10,612] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default1]:[2022-09-07 22:08:10,719] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-07 22:08:10,720] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default3]:[2022-09-07 22:08:10,812] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-07 22:08:10,812] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default2]:[2022-09-07 22:08:10,851] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-07 22:08:10,851] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default2]:[2022-09-07 22:08:10,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-07 22:08:10,797] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default0]:[2022-09-07 22:08:10,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-07 22:08:10,808] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default5]:[2022-09-07 22:08:10,877] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-07 22:08:10,877] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default7]:[2022-09-07 22:08:10,951] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-07 22:08:10,951] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default4]:[2022-09-07 22:08:10,976] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-07 22:08:10,976] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default3]:[2022-09-07 22:08:10,951] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-07 22:08:10,951] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default2]:[2022-09-07 22:08:10,992] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-07 22:08:10,992] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default0]:[2022-09-07 22:08:10,912] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-07 22:08:10,912] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default3]:[2022-09-07 22:08:11,055] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-07 22:08:11,056] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default6]:[2022-09-07 22:08:11,003] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-07 22:08:11,003] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default6]:[2022-09-07 22:08:11,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-07 22:08:11,077] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default4]:[2022-09-07 22:08:11,038] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-07 22:08:11,038] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default1]:[2022-09-07 22:08:11,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-07 22:08:11,077] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default7]:[2022-09-07 22:08:11,102] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-07 22:08:11,102] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default4]:[2022-09-07 22:08:11,103] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-07 22:08:11,103] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default0]:[2022-09-07 22:08:11,128] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-07 22:08:11,128] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default3]:[2022-09-07 22:08:11,116] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-07 22:08:11,116] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default1]:[2022-09-07 22:08:11,135] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-07 22:08:11,136] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default0]:[2022-09-07 22:08:11,184] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-07 22:08:11,184] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default6]:[2022-09-07 22:08:11,162] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-07 22:08:11,162] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default1]:[2022-09-07 22:08:11,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-07 22:08:11,156] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default4]:[2022-09-07 22:08:11,239] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-07 22:08:11,239] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default1]:[2022-09-07 22:08:11,214] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-07 22:08:11,214] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default3]:[2022-09-07 22:08:11,278] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-07 22:08:11,278] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default7]:[2022-09-07 22:08:11,227] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-07 22:08:11,227] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default3]:[2022-09-07 22:08:11,320] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-07 22:08:11,320] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default6]:[2022-09-07 22:08:11,368] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-07 22:08:11,368] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default7]:[2022-09-07 22:08:11,322] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-07 22:08:11,322] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default2]:[2022-09-07 22:08:11,338] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-07 22:08:11,338] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default2]:[2022-09-07 22:08:11,344] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-07 22:08:11,345] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default4]:[2022-09-07 22:08:11,299] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-07 22:08:11,300] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default3]:[2022-09-07 22:08:11,383] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-07 22:08:11,383] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default3]:[2022-09-07 22:08:11,374] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-07 22:08:11,374] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default5]:[2022-09-07 22:08:11,454] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-07 22:08:11,454] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default6]:[2022-09-07 22:08:11,451] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-07 22:08:11,451] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default3]:[2022-09-07 22:08:11,433] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-07 22:08:11,433] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default2]:[2022-09-07 22:08:11,402] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-07 22:08:11,402] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default6]:[2022-09-07 22:08:11,408] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-07 22:08:11,409] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default2]:[2022-09-07 22:08:11,501] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-07 22:08:11,501] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default5]:[2022-09-07 22:08:11,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-07 22:08:11,489] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default7]:[2022-09-07 22:08:11,563] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-07 22:08:11,563] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default4]:[2022-09-07 22:08:11,532] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-07 22:08:11,532] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default4]:[2022-09-07 22:08:11,551] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-07 22:08:11,551] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default6]:[2022-09-07 22:08:11,531] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-07 22:08:11,531] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default5]:[2022-09-07 22:08:11,505] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-07 22:08:11,505] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default7]:[2022-09-07 22:08:11,553] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-07 22:08:11,554] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default2]:[2022-09-07 22:08:11,601] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-07 22:08:11,601] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default7]:[2022-09-07 22:08:11,562] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-07 22:08:11,563] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default3]:[2022-09-07 22:08:11,551] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-07 22:08:11,551] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default2]:[2022-09-07 22:08:11,569] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-07 22:08:11,569] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default2]:[2022-09-07 22:08:11,604] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-07 22:08:11,604] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default1]:[2022-09-07 22:08:11,571] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-07 22:08:11,571] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default1]:[2022-09-07 22:08:11,617] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-07 22:08:11,617] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default1]:[2022-09-07 22:08:11,597] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-07 22:08:11,597] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default6]:[2022-09-07 22:08:11,616] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-07 22:08:11,616] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default6]:[2022-09-07 22:08:11,658] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-07 22:08:11,658] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default5]:[2022-09-07 22:08:11,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-07 22:08:11,670] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default5]:[2022-09-07 22:08:11,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-07 22:08:11,595] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default6]:[2022-09-07 22:08:11,601] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-07 22:08:11,601] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default4]:[2022-09-07 22:08:11,626] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-07 22:08:11,626] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default4]:[2022-09-07 22:08:11,623] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-07 22:08:11,623] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default1]:[2022-09-07 22:08:11,651] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-07 22:08:11,651] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default6]:[2022-09-07 22:08:11,667] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-07 22:08:11,668] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default5]:[2022-09-07 22:08:11,660] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-07 22:08:11,661] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default5]:[2022-09-07 22:08:11,730] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-07 22:08:11,730] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default5]:[2022-09-07 22:08:11,673] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-07 22:08:11,674] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default0]:[2022-09-07 22:08:11,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-07 22:08:11,757] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default7]:[2022-09-07 22:08:11,691] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-07 22:08:11,691] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default7]:[2022-09-07 22:08:11,759] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-07 22:08:11,760] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default5]:[2022-09-07 22:08:11,706] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-07 22:08:11,706] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default0]:[2022-09-07 22:08:11,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-07 22:08:11,797] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default2]:[2022-09-07 22:08:11,872] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-07 22:08:11,872] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default0]:[2022-09-07 22:08:11,841] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-07 22:08:11,841] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default1]:[2022-09-07 22:08:11,816] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-07 22:08:11,816] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default2]:[2022-09-07 22:08:11,915] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-07 22:08:11,915] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default6]:[2022-09-07 22:08:11,968] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-07 22:08:11,968] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default5]:[2022-09-07 22:08:11,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-07 22:08:11,994] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default2]:[2022-09-07 22:08:12,002] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-07 22:08:12,002] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default1]:[2022-09-07 22:08:12,048] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-07 22:08:12,048] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default2]:[2022-09-07 22:08:11,998] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-07 22:08:11,998] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default4]:[2022-09-07 22:08:12,027] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-07 22:08:12,028] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default3]:[2022-09-07 22:08:12,187] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-07 22:08:12,187] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default3]:[2022-09-07 22:08:12,244] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-07 22:08:12,244] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default0]:[2022-09-07 22:08:12,324] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-07 22:08:12,324] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default0]:[2022-09-07 22:08:12,339] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-07 22:08:12,339] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default1]:[2022-09-07 22:08:12,331] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-07 22:08:12,331] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default7]:[2022-09-07 22:08:12,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-07 22:08:12,336] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default2]:[2022-09-07 22:08:12,362] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-07 22:08:12,362] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default1]:[2022-09-07 22:08:12,390] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-07 22:08:12,390] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default5]:[2022-09-07 22:08:12,454] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-07 22:08:12,454] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default4]:[2022-09-07 22:08:12,409] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-07 22:08:12,410] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default3]:[2022-09-07 22:08:12,448] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-07 22:08:12,449] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default6]:[2022-09-07 22:08:12,533] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-07 22:08:12,533] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default6]:[2022-09-07 22:08:12,514] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-07 22:08:12,514] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default2]:[2022-09-07 22:08:12,541] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-07 22:08:12,541] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default0]:[2022-09-07 22:08:12,535] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-07 22:08:12,535] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default4]:[2022-09-07 22:08:12,591] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-07 22:08:12,591] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default5]:[2022-09-07 22:08:12,505] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-07 22:08:12,505] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default5]:[2022-09-07 22:08:12,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-07 22:08:12,541] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default4]:[2022-09-07 22:08:12,521] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-07 22:08:12,522] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default1]:[2022-09-07 22:08:12,539] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-07 22:08:12,539] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default4]:[2022-09-07 22:08:12,626] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-07 22:08:12,626] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default1]:[2022-09-07 22:08:12,566] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-07 22:08:12,567] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default3]:[2022-09-07 22:08:12,658] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-07 22:08:12,658] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default3]:[2022-09-07 22:08:12,630] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-07 22:08:12,630] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default7]:[2022-09-07 22:08:12,669] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-07 22:08:12,669] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default2]:[2022-09-07 22:08:12,684] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-07 22:08:12,684] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default2]:[2022-09-07 22:08:12,697] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-07 22:08:12,697] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default3]:[2022-09-07 22:08:12,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-07 22:08:12,672] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default7]:[2022-09-07 22:08:12,718] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-07 22:08:12,718] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default7]:[2022-09-07 22:08:12,739] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-07 22:08:12,739] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default7]:[2022-09-07 22:08:12,744] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-07 22:08:12,745] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default7]:[2022-09-07 22:08:12,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-07 22:08:12,823] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default7]:[2022-09-07 22:08:12,810] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-07 22:08:12,810] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default0]:[2022-09-07 22:08:12,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-07 22:08:12,831] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default6]:[2022-09-07 22:08:13,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-07 22:08:13,052] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default2]:[2022-09-07 22:08:12,987] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-07 22:08:12,987] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default7]:[2022-09-07 22:08:13,054] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-07 22:08:13,054] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default6]:[2022-09-07 22:08:12,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-07 22:08:12,980] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default3]:[2022-09-07 22:08:13,019] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-07 22:08:13,019] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default3]:[2022-09-07 22:08:13,073] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-07 22:08:13,073] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default5]:[2022-09-07 22:08:13,100] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-07 22:08:13,101] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default2]:[2022-09-07 22:08:13,216] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-07 22:08:13,216] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default1]:[2022-09-07 22:08:13,222] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-07 22:08:13,222] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default7]:[2022-09-07 22:08:13,355] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-07 22:08:13,355] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default4]:[2022-09-07 22:08:13,473] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-07 22:08:13,473] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default0]:[2022-09-07 22:08:13,492] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-07 22:08:13,492] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default5]:[2022-09-07 22:08:13,563] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-07 22:08:13,564] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default7]:[2022-09-07 22:08:13,524] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-07 22:08:13,524] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default7]:[2022-09-07 22:08:13,518] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-07 22:08:13,518] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default6]:[2022-09-07 22:08:13,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-07 22:08:13,636] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default5]:[2022-09-07 22:08:13,627] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-07 22:08:13,627] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default4]:[2022-09-07 22:08:13,620] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-07 22:08:13,620] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default6]:[2022-09-07 22:08:13,663] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-07 22:08:13,663] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default0]:[2022-09-07 22:08:13,605] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-07 22:08:13,605] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default1]:[2022-09-07 22:08:13,666] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-07 22:08:13,666] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default3]:[2022-09-07 22:08:13,680] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-07 22:08:13,680] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default2]:[2022-09-07 22:08:13,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-07 22:08:13,700] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default0]:[2022-09-07 22:08:13,684] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-07 22:08:13,684] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default1]:[2022-09-07 22:08:13,691] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-07 22:08:13,691] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default0]:[2022-09-07 22:08:13,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-07 22:08:13,887] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default5]:[2022-09-07 22:08:13,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-07 22:08:13,800] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default3]:[2022-09-07 22:08:13,881] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-07 22:08:13,881] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default6]:[2022-09-07 22:08:13,895] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-07 22:08:13,895] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default7]:[2022-09-07 22:08:14,131] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-07 22:08:14,132] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default4]:[2022-09-07 22:08:14,176] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-07 22:08:14,176] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default0]:[2022-09-07 22:08:14,278] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-07 22:08:14,278] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default2]:[2022-09-07 22:08:14,688] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-07 22:08:14,688] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default5]:[2022-09-07 22:08:14,644] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-07 22:08:14,644] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default1]:[2022-09-07 22:08:14,657] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-07 22:08:14,657] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default5]:[2022-09-07 22:08:14,779] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-07 22:08:14,780] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default4]:[2022-09-07 22:08:14,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-07 22:08:14,782] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default4]:[2022-09-07 22:08:14,907] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-07 22:08:14,907] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default6]:[2022-09-07 22:08:14,900] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-07 22:08:14,901] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default7]:[2022-09-07 22:08:14,983] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-07 22:08:14,984] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default3]:[2022-09-07 22:08:15,065] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-07 22:08:15,065] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default4]:[2022-09-07 22:08:15,226] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-07 22:08:15,226] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default5]:[2022-09-07 22:08:15,286] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-07 22:08:15,286] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default4]:[2022-09-07 22:08:15,317] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-07 22:08:15,318] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default2]:[2022-09-07 22:08:15,459] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-07 22:08:15,459] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default0]:[2022-09-07 22:08:15,428] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-07 22:08:15,428] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default5]:[2022-09-07 22:08:15,587] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-07 22:08:15,587] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default7]:[2022-09-07 22:08:15,780] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-07 22:08:15,780] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default3]:[2022-09-07 22:08:15,745] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-07 22:08:15,746] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default0]:[2022-09-07 22:08:15,929] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-07 22:08:15,929] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default2]:[2022-09-07 22:08:16,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-07 22:08:16,043] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default6]:[2022-09-07 22:08:16,151] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-07 22:08:16,151] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default3]:[2022-09-07 22:08:16,197] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-07 22:08:16,197] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default4]:[2022-09-07 22:08:16,333] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-07 22:08:16,333] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default5]:[2022-09-07 22:08:16,357] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-07 22:08:16,357] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default1]:[2022-09-07 22:08:16,398] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-07 22:08:16,398] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default4]:[2022-09-07 22:08:16,446] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-07 22:08:16,447] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default7]:[2022-09-07 22:08:16,565] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-07 22:08:16,565] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default0]:[2022-09-07 22:08:16,589] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-07 22:08:16,589] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default6]:[2022-09-07 22:08:16,732] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-07 22:08:16,733] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default6]:[2022-09-07 22:08:16,789] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-07 22:08:16,790] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default0]:[2022-09-07 22:08:16,988] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-07 22:08:16,988] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default7]:[2022-09-07 22:08:17,068] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-07 22:08:17,068] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default5]:[2022-09-07 22:08:17,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-07 22:08:17,043] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default6]:[2022-09-07 22:08:17,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default1]:[2022-09-07 22:08:17,307] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default6]:[2022-09-07 22:08:17,262] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default1]:[2022-09-07 22:08:17,308] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default7]:[2022-09-07 22:08:17,401] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-07 22:08:17,401] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default3]:[2022-09-07 22:08:17,454] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-07 22:08:17,454] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default3]:[2022-09-07 22:08:17,386] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-07 22:08:17,386] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default0]:[2022-09-07 22:08:17,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-07 22:08:17,407] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default2]:[2022-09-07 22:08:17,521] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-07 22:08:17,521] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default0]:[2022-09-07 22:08:17,581] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-07 22:08:17,581] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default5]:[2022-09-07 22:08:17,634] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-07 22:08:17,634] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default2]:[2022-09-07 22:08:17,673] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-07 22:08:17,673] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default0]:[2022-09-07 22:08:17,794] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-07 22:08:17,801] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default4]:[2022-09-07 22:08:17,862] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-07 22:08:17,862] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default6]:[2022-09-07 22:08:17,864] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-07 22:08:17,864] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default0]:[2022-09-07 22:08:17,845] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-07 22:08:17,846] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default7]:[2022-09-07 22:08:17,940] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-07 22:08:17,940] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default1]:[2022-09-07 22:08:17,923] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-07 22:08:17,923] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default6]:[2022-09-07 22:08:18,141] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-07 22:08:18,141] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default4]:[2022-09-07 22:08:18,250] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-07 22:08:18,250] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default1]:[2022-09-07 22:08:18,226] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-07 22:08:18,226] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default4]:[2022-09-07 22:08:18,410] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-07 22:08:18,410] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default5]:[2022-09-07 22:08:18,507] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-07 22:08:18,507] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default5]:[2022-09-07 22:08:18,512] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-07 22:08:18,512] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default1]:[2022-09-07 22:08:18,756] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-07 22:08:18,756] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default3]:[2022-09-07 22:08:18,771] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-07 22:08:18,771] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default5]:[2022-09-07 22:08:19,142] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-07 22:08:19,142] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default2]:[2022-09-07 22:08:19,126] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-07 22:08:19,126] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default0]:[2022-09-07 22:08:19,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-07 22:08:19,186] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default1]:[2022-09-07 22:08:19,245] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-07 22:08:19,245] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default3]:[2022-09-07 22:08:19,237] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-07 22:08:19,238] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default3]:[2022-09-07 22:08:19,257] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-07 22:08:19,257] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default5]:[2022-09-07 22:08:19,260] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-07 22:08:19,260] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default2]:[2022-09-07 22:08:19,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-07 22:08:19,351] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default1]:[2022-09-07 22:08:19,394] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-07 22:08:19,395] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default4]:[2022-09-07 22:08:19,374] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-07 22:08:19,374] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default2]:[2022-09-07 22:08:19,428] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-07 22:08:19,428] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default6]:[2022-09-07 22:08:19,500] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-07 22:08:19,500] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default1]:[2022-09-07 22:08:19,562] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-07 22:08:19,562] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default2]:[2022-09-07 22:08:19,558] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-07 22:08:19,558] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default4]:[2022-09-07 22:08:19,646] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-07 22:08:19,647] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default3]:[2022-09-07 22:08:19,699] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-07 22:08:19,699] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default1]:[2022-09-07 22:08:19,839] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-07 22:08:19,839] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default1]:[2022-09-07 22:08:19,961] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-07 22:08:19,962] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default2]:[2022-09-07 22:08:19,978] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-07 22:08:19,978] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default3]:[2022-09-07 22:08:20,058] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-07 22:08:20,058] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default0]:[2022-09-07 22:08:20,132] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-07 22:08:20,132] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default6]:[2022-09-07 22:08:20,094] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-07 22:08:20,094] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default2]:[2022-09-07 22:08:20,183] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-07 22:08:20,183] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default7]:[2022-09-07 22:08:20,238] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-07 22:08:20,238] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default0]:[2022-09-07 22:08:20,225] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-07 22:08:20,225] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default6]:[2022-09-07 22:08:20,280] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-07 22:08:20,281] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default6]:[2022-09-07 22:08:20,288] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-07 22:08:20,288] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default7]:[2022-09-07 22:08:20,289] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-07 22:08:20,290] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default7]:[2022-09-07 22:08:20,474] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-07 22:08:20,474] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default3]:[2022-09-07 22:08:21,461] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-07 22:08:21,461] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default7]:[2022-09-07 22:08:21,408] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-07 22:08:21,409] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default0]:[2022-09-07 22:08:21,539] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-07 22:08:21,539] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default1]:[2022-09-07 22:08:21,503] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-07 22:08:21,504] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default2]:[2022-09-07 22:08:21,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-07 22:08:21,595] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default2]:[2022-09-07 22:08:21,703] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-07 22:08:21,704] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default4]:[2022-09-07 22:08:21,710] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-07 22:08:21,710] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default5]:[2022-09-07 22:08:21,836] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-07 22:08:21,836] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default4]:[2022-09-07 22:08:21,928] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-07 22:08:21,928] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default3]:[2022-09-07 22:08:22,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-07 22:08:22,016] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default5]:[2022-09-07 22:08:22,144] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-07 22:08:22,144] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default6]:[2022-09-07 22:08:22,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-07 22:08:22,310] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default1]:[2022-09-07 22:08:22,628] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-07 22:08:22,628] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default0]:[2022-09-07 22:08:22,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-07 22:08:22,596] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default7]:[2022-09-07 22:08:22,804] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-07 22:08:22,804] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default6]:[2022-09-07 22:08:23,185] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-07 22:08:23,185] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default7]:[2022-09-07 22:08:24,526] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-07 22:08:24,526] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default3]:[2022-09-07 22:08:24,646] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-07 22:08:24,646] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default6]:[2022-09-07 22:08:24,619] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-07 22:08:24,619] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default7]:[2022-09-07 22:08:24,789] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-07 22:08:24,789] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default5]:[2022-09-07 22:08:25,523] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-07 22:08:25,523] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default4]:[2022-09-07 22:08:25,664] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-07 22:08:25,664] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:time (ms) | save-checkpoint: 32994.71 -[default6]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,237] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-07 22:08:27,237] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]: successfully saved checkpoint at iteration 5 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-07 22:08:27,285] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default4]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default0]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default5]:[2022-09-07 22:08:27,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default6]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default1]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default3]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default2]:[2022-09-07 22:08:27,285] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step5 is ready now! -[default7]: iteration 6/ 3100 | consumed samples: 12288 | consumed tokens: 25165824 | elapsed time per iteration (s): 178.10 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.847781E+00 | grad norm: 2.680 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.499 | TFLOPs: 117.39 | -srun: Job step aborted: Waiting up to 62 seconds for job step to finish. -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107095 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94794 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107096 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89800 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94795 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94727 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94728 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 105801 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87328 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 105802 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81868 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94729 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87329 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 409861 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 93915 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89622 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81869 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94796 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 92721 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94730 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86833 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 93916 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107097 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 409862 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114051 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89623 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87330 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107209 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 92722 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94772 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87094 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94797 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86834 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107098 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114052 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613839 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81870 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94773 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94798 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 97564 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89624 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107099 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87095 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 93917 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114053 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 105803 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 409863 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94799 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 92723 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87331 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86835 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614650 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107100 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114054 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 93918 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94731 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 609236 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89801 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 123810 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613840 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81871 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614184 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613590 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94800 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87332 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94774 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107101 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 164604 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 97565 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114055 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87096 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89802 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 609237 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81872 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614486 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 93919 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614651 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 123811 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94801 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87333 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107102 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613591 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114056 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89625 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613749 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 409864 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614185 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 92724 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 164605 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89803 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 93920 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613841 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81873 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 123812 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614487 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87334 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107210 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614599 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614474 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86836 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94732 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 164606 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114057 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81760 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613750 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 93921 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89804 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81874 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 92725 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 609238 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89626 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86534 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 409865 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 82144 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107211 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 123813 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614600 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614475 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86946 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114058 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 97566 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 93922 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87335 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89805 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 164607 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81875 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613592 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86535 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81761 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614186 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 92726 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 409866 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 82145 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 123814 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107212 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87097 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 101764 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89806 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94775 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 164608 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 409867 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614488 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86947 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89627 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107213 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 92727 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87098 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86837 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 123815 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614652 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613751 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89807 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 164609 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 409868 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 101765 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614476 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 92728 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86838 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94776 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107214 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87099 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 123816 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94733 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89628 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613842 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86536 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 82146 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613752 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 97567 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 164610 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614477 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81762 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86839 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 101766 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107215 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87100 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94777 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89629 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 609239 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 123817 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613753 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86948 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 105804 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94734 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 82147 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 107216 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 87101 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 164611 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86840 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614601 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 101767 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94778 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614187 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613754 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614653 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89634 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 101768 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 94779 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613593 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613755 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 105805 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614654 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89635 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81763 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86537 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 101769 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613843 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613756 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 105806 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614489 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614655 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614602 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 101770 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 105807 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 97568 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614656 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614490 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614478 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613594 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89636 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 101771 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614603 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86949 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 105808 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614657 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614491 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613844 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86538 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81764 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614188 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613595 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86950 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614604 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613845 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613596 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614492 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 609240 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86951 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86539 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614189 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614605 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 97569 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613597 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614494 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 82148 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613846 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86952 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86540 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614479 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614190 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614606 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 97570 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86953 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 86541 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 82149 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614191 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614480 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 97571 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 614481 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 609241 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89637 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 82150 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81765 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 609242 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 82151 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 609243 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89638 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89639 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81766 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89640 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 89641 closing signal SIGTERM -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613737 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 81767 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613738 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613739 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613740 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613741 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613742 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613743 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 613744 closing signal SIGTERM -slurmstepd: error: *** STEP 1096018.0 ON jean-zay-iam03 CANCELLED AT 2022-09-07T22:11:46 *** -WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 95437 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 95438 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 95439 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 95440 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 95441 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 95442 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 95443 closing signal SIGTERM -WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 95444 closing signal SIGTERM -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 614436 got signal: 15 - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 95398 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 87289 got signal: 15 - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 613698 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 86795 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 107167 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 94756 got signal: 15 - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 614561 got signal: 15 - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 114013 got signal: 15 - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 164564 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 92683 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) - main() -torch.distributed.elastic.multiprocessing.api.SignalException: Process 97525 got signal: 15 - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 86496 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 82105 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 101726 got signal: 15 - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - elastic_launch( - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - exec(code, run_globals) - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 613551 got signal: 15 - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 94730 got signal: 15 - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 94688 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code -torch.distributed.elastic.multiprocessing.api.SignalException: Process 81722 got signal: 15 - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - result = self._invoke_run(role) - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - exec(code, run_globals) - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 86906 got signal: 15 - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - main() - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - run(args) - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - elastic_launch( - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - elastic_launch( - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - result = f(*args, **kwargs) - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 409823 got signal: 15 - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - elastic_launch( - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 81830 got signal: 15 - return launch_agent(self._config, self._entrypoint, list(args)) - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper -torch.distributed.elastic.multiprocessing.api.SignalException: Process 613710 got signal: 15 - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 93876 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 609198 got signal: 15 - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 87056 got signal: 15 - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run -torch.distributed.elastic.multiprocessing.api.SignalException: Process 614612 got signal: 15 - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 613801 got signal: 15 - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent -torch.distributed.elastic.multiprocessing.api.SignalException: Process 123772 got signal: 15 - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 614146 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 89762 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 89582 got signal: 15 - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 107057 got signal: 15 - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 614448 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 105763 got signal: 15 -Traceback (most recent call last): - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 194, in _run_module_as_main - return _run_code(code, main_globals, None, - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/runpy.py", line 87, in _run_code - exec(code, run_globals) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 765, in - main() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 345, in wrapper - return f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 761, in main - run(args) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run - elastic_launch( - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 236, in launch_agent - result = agent.run() - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/metrics/api.py", line 125, in wrapper - result = f(*args, **kwargs) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 709, in run - result = self._invoke_run(role) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 850, in _invoke_run - time.sleep(monitor_interval) - File "/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 60, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 89595 got signal: 15 -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -WARNING:__main__: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[default0]:Offline mode: forcing local_files_only=True -[default0]:Offline mode: forcing local_files_only=True -[default0]:using world size: 288, data-parallel-size: 4, tensor-model-parallel size: 1, pipeline-model-parallel size: 72 -[default0]:accumulate and all-reduce gradients in fp32 for bfloat16 data type. -[default0]:using torch.bfloat16 for parameters ... -[default0]:------------------------ arguments ------------------------ -[default0]: abort_on_unmet_fused_kernel_constraints ......... True -[default0]: accumulate_allreduce_grads_in_fp32 .............. True -[default0]: adam_beta1 ...................................... 0.9 -[default0]: adam_beta2 ...................................... 0.95 -[default0]: adam_eps ........................................ 1e-08 -[default0]: adlr_autoresume ................................. False -[default0]: adlr_autoresume_interval ........................ 1000 -[default0]: apply_query_key_layer_scaling ................... True -[default0]: apply_residual_connection_post_layernorm ........ False -[default0]: attention_dropout ............................... 0.1 -[default0]: attention_softmax_in_fp32 ....................... False -[default0]: bert_binary_head ................................ True -[default0]: bert_load ....................................... None -[default0]: bf16 ............................................ True -[default0]: bias_dropout_fusion ............................. True -[default0]: bias_gelu_fusion ................................ True -[default0]: biencoder_projection_dim ........................ 0 -[default0]: biencoder_shared_query_context_model ............ False -[default0]: block_data_path ................................. None -[default0]: checkpoint_activations .......................... True -[default0]: checkpoint_in_cpu ............................... False -[default0]: checkpoint_num_layers ........................... 1 -[default0]: clip_grad ....................................... 1.0 -[default0]: codecarbon_dir .................................. None -[default0]: consumed_train_samples .......................... 0 -[default0]: consumed_train_tokens ........................... 0 -[default0]: consumed_valid_samples .......................... 0 -[default0]: contigious_checkpointing ........................ False -[default0]: cpu_optimizer ................................... False -[default0]: cpu_torch_adam .................................. False -[default0]: curriculum_learning ............................. False -[default0]: data_impl ....................................... mmap -[default0]: data_parallel_size .............................. 4 -[default0]: data_path ....................................... None -[default0]: dataloader_type ................................. single -[default0]: DDP_impl ........................................ local -[default0]: decoder_seq_length .............................. None -[default0]: deepscale ....................................... False -[default0]: deepscale_config ................................ None -[default0]: deepspeed ....................................... True -[default0]: deepspeed_activation_checkpointing .............. True -[default0]: deepspeed_config ................................ ./ds_config.1096241.json -[default0]: deepspeed_mpi ................................... False -[default0]: distribute_checkpointed_activations ............. False -[default0]: distributed_backend ............................. nccl -[default0]: embed_layernorm ................................. True -[default0]: embedding_path .................................. None -[default0]: encoder_seq_length .............................. 2048 -[default0]: eod_mask_loss ................................... False -[default0]: eval_interval ................................... 250 -[default0]: eval_iters ...................................... 1 -[default0]: eval_only ....................................... None -[default0]: evidence_data_path .............................. None -[default0]: exit_duration_in_mins ........................... 5990 -[default0]: exit_interval ................................... None -[default0]: ffn_hidden_size ................................. 57344 -[default0]: finetune ........................................ False -[default0]: fp16 ............................................ False -[default0]: fp16_lm_cross_entropy ........................... False -[default0]: fp32_residual_connection ........................ False -[default0]: gigaflos_no_embeds .............................. 0 -[default0]: global_batch_size ............................... 2048 -[default0]: glu_activation .................................. None -[default0]: hidden_dropout .................................. 0.1 -[default0]: hidden_size ..................................... 14336 -[default0]: hysteresis ...................................... 2 -[default0]: ict_head_size ................................... None -[default0]: ict_load ........................................ None -[default0]: img_dim ......................................... 224 -[default0]: indexer_batch_size .............................. 128 -[default0]: indexer_log_interval ............................ 1000 -[default0]: inference ....................................... False -[default0]: init_method_std ................................. 0.0048 -[default0]: init_method_xavier_uniform ...................... False -[default0]: initial_loss_scale .............................. 4294967296 -[default0]: kill_switch_path ................................ /gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/kill-switch-tr13-176B-mtf -[default0]: kv_channels ..................................... 128 -[default0]: layernorm_epsilon ............................... 1e-05 -[default0]: lazy_mpu_init ................................... None -[default0]: load ............................................ /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]: local_rank ...................................... None -[default0]: log_batch_size_to_tensorboard ................... True -[default0]: log_interval .................................... 1 -[default0]: log_learning_rate_to_tensorboard ................ True -[default0]: log_level ....................................... None -[default0]: log_level_replica ............................... None -[default0]: log_loss_scale_to_tensorboard ................... True -[default0]: log_num_zeros_in_grad ........................... False -[default0]: log_params_norm ................................. False -[default0]: log_path ........................................ None -[default0]: log_timers_to_tensorboard ....................... True -[default0]: log_validation_ppl_to_tensorboard ............... True -[default0]: loss_on_targets_only ............................ False -[default0]: loss_scale ...................................... None -[default0]: loss_scale_window ............................... 1000 -[default0]: lr .............................................. 2e-05 -[default0]: lr_decay_iters .................................. None -[default0]: lr_decay_samples ................................ None -[default0]: lr_decay_style .................................. constant -[default0]: lr_decay_tokens ................................. None -[default0]: lr_warmup_fraction .............................. None -[default0]: lr_warmup_iters ................................. 0 -[default0]: lr_warmup_samples ............................... 0 -[default0]: make_vocab_size_divisible_by .................... 128 -[default0]: mask_prob ....................................... 0.15 -[default0]: masked_softmax_fusion ........................... True -[default0]: max_position_embeddings ......................... 2048 -[default0]: mean_noise_span_length .......................... None -[default0]: memory_centric_tiled_linear ..................... False -[default0]: merge_file ...................................... None -[default0]: micro_batch_size ................................ 1 -[default0]: min_loss_scale .................................. 1.0 -[default0]: min_lr .......................................... 0.0 -[default0]: mmap_warmup ..................................... False -[default0]: no_load_optim ................................... None -[default0]: no_load_rng ..................................... None -[default0]: no_save_optim ................................... None -[default0]: no_save_rng ..................................... None -[default0]: noise_density ................................... None -[default0]: norm_target_loss ................................ True -[default0]: num_attention_heads ............................. 112 -[default0]: num_channels .................................... 3 -[default0]: num_classes ..................................... 1000 -[default0]: num_layers ...................................... 70 -[default0]: num_layers_per_virtual_pipeline_stage ........... None -[default0]: num_workers ..................................... 2 -[default0]: onnx_safe ....................................... None -[default0]: openai_gelu ..................................... False -[default0]: optimizer ....................................... adam -[default0]: override_lr_scheduler ........................... False -[default0]: pad_vocab_size_to ............................... 250880 -[default0]: params_dtype .................................... torch.bfloat16 -[default0]: partition_activations ........................... False -[default0]: patch_dim ....................................... 16 -[default0]: pipeline_model_parallel_size .................... 72 -[default0]: position_embedding_type ......................... PositionEmbeddingType.alibi -[default0]: pp_partition_method ............................. type:transformer|embedding -[default0]: prefixlm ........................................ False -[default0]: profile_backward ................................ False -[default0]: query_in_block_prob ............................. 0.1 -[default0]: rampup_batch_size ............................... None -[default0]: rank ............................................ 0 -[default0]: remote_device ................................... none -[default0]: reset_attention_mask ............................ False -[default0]: reset_position_ids .............................. False -[default0]: reset_progress .................................. None -[default0]: retriever_report_topk_accuracies ................ [] -[default0]: retriever_score_scaling ......................... False -[default0]: retriever_seq_length ............................ 256 -[default0]: reweight_loss_based_on_position_frequency ....... False -[default0]: sample_rate ..................................... 1.0 -[default0]: save ............................................ /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]: save_interval ................................... 249 -[default0]: scatter_gather_tensors_in_pipeline .............. True -[default0]: scattered_embeddings ............................ False -[default0]: seed ............................................ 42 -[default0]: seq_length ...................................... 2048 -[default0]: sgd_momentum .................................... 0.9 -[default0]: short_seq_prob .................................. 0.1 -[default0]: skip_train_iteration_range ...................... None -[default0]: split ........................................... None -[default0]: split_transformers .............................. False -[default0]: sync_tp_duplicated_parameters ................... True -[default0]: synchronize_each_layer .......................... False -[default0]: tensor_model_parallel_size ...................... 1 -[default0]: tensorboard_dir ................................. /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/tr13-176B-ml-t0-logs/tensorboard/xp3zzlossseq -[default0]: tensorboard_log_interval ........................ 1 -[default0]: tensorboard_queue_size .......................... 5 -[default0]: test_weighted_split_paths ....................... None -[default0]: test_weighted_split_paths_path .................. None -[default0]: tile_factor ..................................... 1 -[default0]: titles_data_path ................................ None -[default0]: tokenizer_name_or_path .......................... bigscience/tokenizer -[default0]: tokenizer_type .................................. PretrainedFromHF -[default0]: train_iters ..................................... None -[default0]: train_samples ................................... 6348800 -[default0]: train_tokens .................................... None -[default0]: train_weighted_split_names ...................... ['train'] -[default0]: train_weighted_split_paths ...................... [['/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_en', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_es', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pt', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fr', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_code', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ar', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_id', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zh', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_hi', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_vi', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ur', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_te', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ta', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bn', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_mr', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sw', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_gu', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pa', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ne', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_yo', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ig', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ny', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zu', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_xh', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sn', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ts', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rw', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_lg', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tn', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_nso', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rn', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ml', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_kn', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_or', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_as', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ln', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_wo', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tum', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ki', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_st', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fon', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_eu', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ca', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ak', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bm', '/gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tw']] -[default0]: train_weighted_split_paths_path ................. None -[default0]: train_weighted_split_splits ..................... [['0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1', '0:1']] -[default0]: train_weighted_split_weights .................... [['0.3932835937', '0.0860451087', '0.0690010451', '0.0660385329', '0.0590120118', '0.046925039', '0.0462116635', '0.0460832559', '0.0441207519', '0.031085057', '0.0192197788', '0.0134582697', '0.0092870269', '0.0083432872', '0.006675271', '0.0056775071', '0.0056177118', '0.0052425885', '0.0039444054', '0.0035346554', '0.0032586031', '0.0027265372', '0.0026422146', '0.00255164', '0.0025298379', '0.0025073947', '0.0024984173', '0.002363918', '0.0023599103', '0.0023015578', '0.0019336484', '0.0017537816', '0.0016577564', '0.0016178395', '0.0015655787', '0.00126548', '0.0012279677', '0.0011625616', '0.0011526224', '0.0011430039', '0.0011329044', '0.0011322632', '0.0011082168', '0.0010830483', '0.0010726282', '0.0010649334']] -[default0]: universal_checkpoint ............................ False -[default0]: use_bnb_optimizer ............................... False -[default0]: use_checkpoint_lr_scheduler ..................... False -[default0]: use_contiguous_buffers_in_ddp ................... True -[default0]: use_cpu_initialization .......................... None -[default0]: use_one_sent_docs ............................... False -[default0]: use_pin_memory .................................. False -[default0]: valid_num_workers ............................... 2 -[default0]: valid_weighted_split_names ...................... ['validation_pretraining'] -[default0]: valid_weighted_split_paths ...................... [['/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ar/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ar_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ca/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ca_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/code/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_code_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/en/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_en_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/es/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_es_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/eu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_eu_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/fr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_fr_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/id/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_id_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-as/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-as_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-bn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-bn_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-gu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-gu_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-hi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-hi_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-kn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-kn_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ml/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ml_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-mr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-mr_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ne/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ne_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-or/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-or_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-pa/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-pa_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ta/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ta_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-te/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-te_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ur/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ur_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/nigercongo-all/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_nigercongo-all_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-en/meg_ds_bigscience_tokenizer_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-zh/meg_ds_bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/pt/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_pt_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/vi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_vi_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zhs/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zhs_text_document', '/gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zht/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zht_text_document']] -[default0]: valid_weighted_split_paths_path ................. None -[default0]: valid_weighted_split_splits ..................... [['0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0', '0.950:1.0']] -[default0]: valid_weighted_split_weights .................... [['0.0330676168743166', '0.011242051312222764', '0.13027200903379185', '0.22171164529099704', '0.10667815627928671', '0.0015595123898173287', '0.13054018439603915', '0.01091803753667153', '0.00011021422347108609', '0.005492381453597748', '0.0004021215011318779', '0.007470068593492175', '0.0006190467776576425', '0.0010335296343329384', '0.0005012010684646179', '0.0006672772956128299', '0.00035928138344705506', '0.0005084433130291778', '0.0021137328219915496', '0.0009129946225980253', '0.0012454301613725426', '0.00031588689199263235', '0.08137213783015229', '0.055293935695898196', '0.04954150576361177', '0.02461641286531197', '0.12091748245519074', '0.0005177025345001541']] -[default0]: virtual_pipeline_model_parallel_size ............ None -[default0]: vocab_extra_ids ................................. 0 -[default0]: vocab_file ...................................... None -[default0]: weight_decay .................................... 0.0001 -[default0]: world_size ...................................... 288 -[default0]: zero_allgather_bucket_size ...................... 0.0 -[default0]: zero_contigious_gradients ....................... False -[default0]: zero_reduce_bucket_size ......................... 0.0 -[default0]: zero_reduce_scatter ............................. False -[default0]: zero_stage ...................................... 0 -[default0]:-------------------- end of arguments --------------------- -[default0]:setting number of micro-batches to constant 512 -[default0]:> building PretrainedFromHF tokenizer ... -[default0]: vocab file is un-used. loading tokenizer from pre-trained model -[default7]:> setting tensorboard ... -[default0]:loading file https://huggingface.co/bigscience/tokenizer/resolve/main/tokenizer.json from cache at /gpfswork/rech/six/commun/models/29d0a41f4527257b8afe6d5495f492dac260318430f18239a42ca5f6dc4487fc.7b0fb8edc2986944ff9b7418149b52d8c4a1354a17d0360deb8974da70c6cc03 -[default0]:loading file https://huggingface.co/bigscience/tokenizer/resolve/main/added_tokens.json from cache at None -[default0]:loading file https://huggingface.co/bigscience/tokenizer/resolve/main/special_tokens_map.json from cache at /gpfswork/rech/six/commun/models/4f03e43bcc54e0721823e6a06b1d197905e2ea79aa7dcc1a0f0fcecc73ce3fb2.9d6cd81ef646692fb1c169a880161ea1cb95f49694f220aced9b704b457e51dd -[default0]:loading file https://huggingface.co/bigscience/tokenizer/resolve/main/tokenizer_config.json from cache at /gpfswork/rech/six/commun/models/9441c67b923ef7a65950a64e31c40f80ed181ba59502981a80f2cd0c438c6432.3c09887250243e50d8de9d10b2a778152434f62a22a95b5f89dbbe79a6eb496a -[default0]: > padded vocab (size: 250680) with 200 dummy tokens (new size: 250880) -[default0]:DeepSpeed general environment info: -[default0]:torch install path ............... ['/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch'] -[default0]:torch version .................... 1.12.0 -[default0]:torch cuda version ............... 11.3 -[default0]:torch hip version ................ None -[default0]:nvcc version ..................... 11.4 -[default0]:deepspeed install path ........... ['/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/deepspeed'] -[default0]:deepspeed info ................... 0.7.1+8b2a6371, 8b2a6371, master -[default0]:deepspeed wheel compiled w. ...... torch 1.12, cuda 11.3 -[default0]:**** Git info for Megatron: git_hash=6c1018f git_branch=mtf-multival **** -[default0]:> initializing torch distributed ... -[default0]:[2022-09-07 22:14:13,111] [INFO] [comm.py:628:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl -[default0]:> initializing tensor model parallel with size 1 -[default0]:> initializing pipeline model parallel with size 72 -[default0]:> setting random seeds to 42 ... -[default0]:[2022-09-07 22:14:21,300] [INFO] [checkpointing.py:226:model_parallel_cuda_manual_seed] > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 2760 and data parallel seed: 42 -[default0]:> compiling dataset index builder ... -[default0]:make: Entering directory '/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/data' -[default0]:make: Nothing to be done for 'default'. -[default0]:make: Leaving directory '/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/data' -[default0]:>>> done with dataset index builder. Compilation time: 0.085 seconds -[default0]:> compiling and loading fused kernels ... -[default0]:Detected CUDA files, patching ldflags -[default0]:Emitting ninja build file /gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... -[default0]:Building extension module scaled_upper_triang_masked_softmax_cuda... -[default0]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default0]:ninja: no work to do. -[default0]:Loading extension module scaled_upper_triang_masked_softmax_cuda... -[default0]:Detected CUDA files, patching ldflags -[default0]:Emitting ninja build file /gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... -[default0]:Building extension module scaled_masked_softmax_cuda... -[default0]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default0]:ninja: no work to do. -[default0]:Loading extension module scaled_masked_softmax_cuda... -[default0]:Detected CUDA files, patching ldflags -[default0]:Emitting ninja build file /gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... -[default0]:Building extension module fused_mix_prec_layer_norm_cuda... -[default0]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default0]:ninja: no work to do. -[default0]:Loading extension module fused_mix_prec_layer_norm_cuda... -[default0]:>>> done with compiling and loading fused kernels. Compilation time: 6.505 seconds -[default0]:time to initialize megatron (seconds): 35.893 -[default0]:[after megatron is initialized] datetime: 2022-09-07 22:14:27 -[default0]:building GPT model ... -[default0]:[2022-09-07 22:14:27,927] [INFO] [utils.py:827:see_memory_usage] Before Building Model -[default0]:[2022-09-07 22:14:27,928] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB -[default0]:[2022-09-07 22:14:27,928] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.63 GB, percent = 6.3% -[default0]:SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None -[default0]:Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=1, data=0, model=0): 4, ProcessCoord(pipe=1, data=1, model=0): 5, ProcessCoord(pipe=1, data=2, model=0): 6, ProcessCoord(pipe=1, data=3, model=0): 7, ProcessCoord(pipe=2, data=0, model=0): 8, ProcessCoord(pipe=2, data=1, model=0): 9, ProcessCoord(pipe=2, data=2, model=0): 10, ProcessCoord(pipe=2, data=3, model=0): 11, ProcessCoord(pipe=3, data=0, model=0): 12, ProcessCoord(pipe=3, data=1, model=0): 13, ProcessCoord(pipe=3, data=2, model=0): 14, ProcessCoord(pipe=3, data=3, model=0): 15, ProcessCoord(pipe=4, data=0, model=0): 16, ProcessCoord(pipe=4, data=1, model=0): 17, ProcessCoord(pipe=4, data=2, model=0): 18, ProcessCoord(pipe=4, data=3, model=0): 19, ProcessCoord(pipe=5, data=0, model=0): 20, ProcessCoord(pipe=5, data=1, model=0): 21, ProcessCoord(pipe=5, data=2, model=0): 22, ProcessCoord(pipe=5, data=3, model=0): 23, ProcessCoord(pipe=6, data=0, model=0): 24, ProcessCoord(pipe=6, data=1, model=0): 25, ProcessCoord(pipe=6, data=2, model=0): 26, ProcessCoord(pipe=6, data=3, model=0): 27, ProcessCoord(pipe=7, data=0, model=0): 28, ProcessCoord(pipe=7, data=1, model=0): 29, ProcessCoord(pipe=7, data=2, model=0): 30, ProcessCoord(pipe=7, data=3, model=0): 31, ProcessCoord(pipe=8, data=0, model=0): 32, ProcessCoord(pipe=8, data=1, model=0): 33, ProcessCoord(pipe=8, data=2, model=0): 34, ProcessCoord(pipe=8, data=3, model=0): 35, ProcessCoord(pipe=9, data=0, model=0): 36, ProcessCoord(pipe=9, data=1, model=0): 37, ProcessCoord(pipe=9, data=2, model=0): 38, ProcessCoord(pipe=9, data=3, model=0): 39, ProcessCoord(pipe=10, data=0, model=0): 40, ProcessCoord(pipe=10, data=1, model=0): 41, ProcessCoord(pipe=10, data=2, model=0): 42, ProcessCoord(pipe=10, data=3, model=0): 43, ProcessCoord(pipe=11, data=0, model=0): 44, ProcessCoord(pipe=11, data=1, model=0): 45, ProcessCoord(pipe=11, data=2, model=0): 46, ProcessCoord(pipe=11, data=3, model=0): 47, ProcessCoord(pipe=12, data=0, model=0): 48, ProcessCoord(pipe=12, data=1, model=0): 49, ProcessCoord(pipe=12, data=2, model=0): 50, ProcessCoord(pipe=12, data=3, model=0): 51, ProcessCoord(pipe=13, data=0, model=0): 52, ProcessCoord(pipe=13, data=1, model=0): 53, ProcessCoord(pipe=13, data=2, model=0): 54, ProcessCoord(pipe=13, data=3, model=0): 55, ProcessCoord(pipe=14, data=0, model=0): 56, ProcessCoord(pipe=14, data=1, model=0): 57, ProcessCoord(pipe=14, data=2, model=0): 58, ProcessCoord(pipe=14, data=3, model=0): 59, ProcessCoord(pipe=15, data=0, model=0): 60, ProcessCoord(pipe=15, data=1, model=0): 61, ProcessCoord(pipe=15, data=2, model=0): 62, ProcessCoord(pipe=15, data=3, model=0): 63, ProcessCoord(pipe=16, data=0, model=0): 64, ProcessCoord(pipe=16, data=1, model=0): 65, ProcessCoord(pipe=16, data=2, model=0): 66, ProcessCoord(pipe=16, data=3, model=0): 67, ProcessCoord(pipe=17, data=0, model=0): 68, ProcessCoord(pipe=17, data=1, model=0): 69, ProcessCoord(pipe=17, data=2, model=0): 70, ProcessCoord(pipe=17, data=3, model=0): 71, ProcessCoord(pipe=18, data=0, model=0): 72, ProcessCoord(pipe=18, data=1, model=0): 73, ProcessCoord(pipe=18, data=2, model=0): 74, ProcessCoord(pipe=18, data=3, model=0): 75, ProcessCoord(pipe=19, data=0, model=0): 76, ProcessCoord(pipe=19, data=1, model=0): 77, ProcessCoord(pipe=19, data=2, model=0): 78, ProcessCoord(pipe=19, data=3, model=0): 79, ProcessCoord(pipe=20, data=0, model=0): 80, ProcessCoord(pipe=20, data=1, model=0): 81, ProcessCoord(pipe=20, data=2, model=0): 82, ProcessCoord(pipe=20, data=3, model=0): 83, ProcessCoord(pipe=21, data=0, model=0): 84, ProcessCoord(pipe=21, data=1, model=0): 85, ProcessCoord(pipe=21, data=2, model=0): 86, ProcessCoord(pipe=21, data=3, model=0): 87, ProcessCoord(pipe=22, data=0, model=0): 88, ProcessCoord(pipe=22, data=1, model=0): 89, ProcessCoord(pipe=22, data=2, model=0): 90, ProcessCoord(pipe=22, data=3, model=0): 91, ProcessCoord(pipe=23, data=0, model=0): 92, ProcessCoord(pipe=23, data=1, model=0): 93, ProcessCoord(pipe=23, data=2, model=0): 94, ProcessCoord(pipe=23, data=3, model=0): 95, ProcessCoord(pipe=24, data=0, model=0): 96, ProcessCoord(pipe=24, data=1, model=0): 97, ProcessCoord(pipe=24, data=2, model=0): 98, ProcessCoord(pipe=24, data=3, model=0): 99, ProcessCoord(pipe=25, data=0, model=0): 100, ProcessCoord(pipe=25, data=1, model=0): 101, ProcessCoord(pipe=25, data=2, model=0): 102, ProcessCoord(pipe=25, data=3, model=0): 103, ProcessCoord(pipe=26, data=0, model=0): 104, ProcessCoord(pipe=26, data=1, model=0): 105, ProcessCoord(pipe=26, data=2, model=0): 106, ProcessCoord(pipe=26, data=3, model=0): 107, ProcessCoord(pipe=27, data=0, model=0): 108, ProcessCoord(pipe=27, data=1, model=0): 109, ProcessCoord(pipe=27, data=2, model=0): 110, ProcessCoord(pipe=27, data=3, model=0): 111, ProcessCoord(pipe=28, data=0, model=0): 112, ProcessCoord(pipe=28, data=1, model=0): 113, ProcessCoord(pipe=28, data=2, model=0): 114, ProcessCoord(pipe=28, data=3, model=0): 115, ProcessCoord(pipe=29, data=0, model=0): 116, ProcessCoord(pipe=29, data=1, model=0): 117, ProcessCoord(pipe=29, data=2, model=0): 118, ProcessCoord(pipe=29, data=3, model=0): 119, ProcessCoord(pipe=30, data=0, model=0): 120, ProcessCoord(pipe=30, data=1, model=0): 121, ProcessCoord(pipe=30, data=2, model=0): 122, ProcessCoord(pipe=30, data=3, model=0): 123, ProcessCoord(pipe=31, data=0, model=0): 124, ProcessCoord(pipe=31, data=1, model=0): 125, ProcessCoord(pipe=31, data=2, model=0): 126, ProcessCoord(pipe=31, data=3, model=0): 127, ProcessCoord(pipe=32, data=0, model=0): 128, ProcessCoord(pipe=32, data=1, model=0): 129, ProcessCoord(pipe=32, data=2, model=0): 130, ProcessCoord(pipe=32, data=3, model=0): 131, ProcessCoord(pipe=33, data=0, model=0): 132, ProcessCoord(pipe=33, data=1, model=0): 133, ProcessCoord(pipe=33, data=2, model=0): 134, ProcessCoord(pipe=33, data=3, model=0): 135, ProcessCoord(pipe=34, data=0, model=0): 136, ProcessCoord(pipe=34, data=1, model=0): 137, ProcessCoord(pipe=34, data=2, model=0): 138, ProcessCoord(pipe=34, data=3, model=0): 139, ProcessCoord(pipe=35, data=0, model=0): 140, ProcessCoord(pipe=35, data=1, model=0): 141, ProcessCoord(pipe=35, data=2, model=0): 142, ProcessCoord(pipe=35, data=3, model=0): 143, ProcessCoord(pipe=36, data=0, model=0): 144, ProcessCoord(pipe=36, data=1, model=0): 145, ProcessCoord(pipe=36, data=2, model=0): 146, ProcessCoord(pipe=36, data=3, model=0): 147, ProcessCoord(pipe=37, data=0, model=0): 148, ProcessCoord(pipe=37, data=1, model=0): 149, ProcessCoord(pipe=37, data=2, model=0): 150, ProcessCoord(pipe=37, data=3, model=0): 151, ProcessCoord(pipe=38, data=0, model=0): 152, ProcessCoord(pipe=38, data=1, model=0): 153, ProcessCoord(pipe=38, data=2, model=0): 154, ProcessCoord(pipe=38, data=3, model=0): 155, ProcessCoord(pipe=39, data=0, model=0): 156, ProcessCoord(pipe=39, data=1, model=0): 157, ProcessCoord(pipe=39, data=2, model=0): 158, ProcessCoord(pipe=39, data=3, model=0): 159, ProcessCoord(pipe=40, data=0, model=0): 160, ProcessCoord(pipe=40, data=1, model=0): 161, ProcessCoord(pipe=40, data=2, model=0): 162, ProcessCoord(pipe=40, data=3, model=0): 163, ProcessCoord(pipe=41, data=0, model=0): 164, ProcessCoord(pipe=41, data=1, model=0): 165, ProcessCoord(pipe=41, data=2, model=0): 166, ProcessCoord(pipe=41, data=3, model=0): 167, ProcessCoord(pipe=42, data=0, model=0): 168, ProcessCoord(pipe=42, data=1, model=0): 169, ProcessCoord(pipe=42, data=2, model=0): 170, ProcessCoord(pipe=42, data=3, model=0): 171, ProcessCoord(pipe=43, data=0, model=0): 172, ProcessCoord(pipe=43, data=1, model=0): 173, ProcessCoord(pipe=43, data=2, model=0): 174, ProcessCoord(pipe=43, data=3, model=0): 175, ProcessCoord(pipe=44, data=0, model=0): 176, ProcessCoord(pipe=44, data=1, model=0): 177, ProcessCoord(pipe=44, data=2, model=0): 178, ProcessCoord(pipe=44, data=3, model=0): 179, ProcessCoord(pipe=45, data=0, model=0): 180, ProcessCoord(pipe=45, data=1, model=0): 181, ProcessCoord(pipe=45, data=2, model=0): 182, ProcessCoord(pipe=45, data=3, model=0): 183, ProcessCoord(pipe=46, data=0, model=0): 184, ProcessCoord(pipe=46, data=1, model=0): 185, ProcessCoord(pipe=46, data=2, model=0): 186, ProcessCoord(pipe=46, data=3, model=0): 187, ProcessCoord(pipe=47, data=0, model=0): 188, ProcessCoord(pipe=47, data=1, model=0): 189, ProcessCoord(pipe=47, data=2, model=0): 190, ProcessCoord(pipe=47, data=3, model=0): 191, ProcessCoord(pipe=48, data=0, model=0): 192, ProcessCoord(pipe=48, data=1, model=0): 193, ProcessCoord(pipe=48, data=2, model=0): 194, ProcessCoord(pipe=48, data=3, model=0): 195, ProcessCoord(pipe=49, data=0, model=0): 196, ProcessCoord(pipe=49, data=1, model=0): 197, ProcessCoord(pipe=49, data=2, model=0): 198, ProcessCoord(pipe=49, data=3, model=0): 199, ProcessCoord(pipe=50, data=0, model=0): 200, ProcessCoord(pipe=50, data=1, model=0): 201, ProcessCoord(pipe=50, data=2, model=0): 202, ProcessCoord(pipe=50, data=3, model=0): 203, ProcessCoord(pipe=51, data=0, model=0): 204, ProcessCoord(pipe=51, data=1, model=0): 205, ProcessCoord(pipe=51, data=2, model=0): 206, ProcessCoord(pipe=51, data=3, model=0): 207, ProcessCoord(pipe=52, data=0, model=0): 208, ProcessCoord(pipe=52, data=1, model=0): 209, ProcessCoord(pipe=52, data=2, model=0): 210, ProcessCoord(pipe=52, data=3, model=0): 211, ProcessCoord(pipe=53, data=0, model=0): 212, ProcessCoord(pipe=53, data=1, model=0): 213, ProcessCoord(pipe=53, data=2, model=0): 214, ProcessCoord(pipe=53, data=3, model=0): 215, ProcessCoord(pipe=54, data=0, model=0): 216, ProcessCoord(pipe=54, data=1, model=0): 217, ProcessCoord(pipe=54, data=2, model=0): 218, ProcessCoord(pipe=54, data=3, model=0): 219, ProcessCoord(pipe=55, data=0, model=0): 220, ProcessCoord(pipe=55, data=1, model=0): 221, ProcessCoord(pipe=55, data=2, model=0): 222, ProcessCoord(pipe=55, data=3, model=0): 223, ProcessCoord(pipe=56, data=0, model=0): 224, ProcessCoord(pipe=56, data=1, model=0): 225, ProcessCoord(pipe=56, data=2, model=0): 226, ProcessCoord(pipe=56, data=3, model=0): 227, ProcessCoord(pipe=57, data=0, model=0): 228, ProcessCoord(pipe=57, data=1, model=0): 229, ProcessCoord(pipe=57, data=2, model=0): 230, ProcessCoord(pipe=57, data=3, model=0): 231, ProcessCoord(pipe=58, data=0, model=0): 232, ProcessCoord(pipe=58, data=1, model=0): 233, ProcessCoord(pipe=58, data=2, model=0): 234, ProcessCoord(pipe=58, data=3, model=0): 235, ProcessCoord(pipe=59, data=0, model=0): 236, ProcessCoord(pipe=59, data=1, model=0): 237, ProcessCoord(pipe=59, data=2, model=0): 238, ProcessCoord(pipe=59, data=3, model=0): 239, ProcessCoord(pipe=60, data=0, model=0): 240, ProcessCoord(pipe=60, data=1, model=0): 241, ProcessCoord(pipe=60, data=2, model=0): 242, ProcessCoord(pipe=60, data=3, model=0): 243, ProcessCoord(pipe=61, data=0, model=0): 244, ProcessCoord(pipe=61, data=1, model=0): 245, ProcessCoord(pipe=61, data=2, model=0): 246, ProcessCoord(pipe=61, data=3, model=0): 247, ProcessCoord(pipe=62, data=0, model=0): 248, ProcessCoord(pipe=62, data=1, model=0): 249, ProcessCoord(pipe=62, data=2, model=0): 250, ProcessCoord(pipe=62, data=3, model=0): 251, ProcessCoord(pipe=63, data=0, model=0): 252, ProcessCoord(pipe=63, data=1, model=0): 253, ProcessCoord(pipe=63, data=2, model=0): 254, ProcessCoord(pipe=63, data=3, model=0): 255, ProcessCoord(pipe=64, data=0, model=0): 256, ProcessCoord(pipe=64, data=1, model=0): 257, ProcessCoord(pipe=64, data=2, model=0): 258, ProcessCoord(pipe=64, data=3, model=0): 259, ProcessCoord(pipe=65, data=0, model=0): 260, ProcessCoord(pipe=65, data=1, model=0): 261, ProcessCoord(pipe=65, data=2, model=0): 262, ProcessCoord(pipe=65, data=3, model=0): 263, ProcessCoord(pipe=66, data=0, model=0): 264, ProcessCoord(pipe=66, data=1, model=0): 265, ProcessCoord(pipe=66, data=2, model=0): 266, ProcessCoord(pipe=66, data=3, model=0): 267, ProcessCoord(pipe=67, data=0, model=0): 268, ProcessCoord(pipe=67, data=1, model=0): 269, ProcessCoord(pipe=67, data=2, model=0): 270, ProcessCoord(pipe=67, data=3, model=0): 271, ProcessCoord(pipe=68, data=0, model=0): 272, ProcessCoord(pipe=68, data=1, model=0): 273, ProcessCoord(pipe=68, data=2, model=0): 274, ProcessCoord(pipe=68, data=3, model=0): 275, ProcessCoord(pipe=69, data=0, model=0): 276, ProcessCoord(pipe=69, data=1, model=0): 277, ProcessCoord(pipe=69, data=2, model=0): 278, ProcessCoord(pipe=69, data=3, model=0): 279, ProcessCoord(pipe=70, data=0, model=0): 280, ProcessCoord(pipe=70, data=1, model=0): 281, ProcessCoord(pipe=70, data=2, model=0): 282, ProcessCoord(pipe=70, data=3, model=0): 283, ProcessCoord(pipe=71, data=0, model=0): 284, ProcessCoord(pipe=71, data=1, model=0): 285, ProcessCoord(pipe=71, data=2, model=0): 286, ProcessCoord(pipe=71, data=3, model=0): 287} -[default0]:[2022-09-07 22:14:31,784] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer|embedding -[default0]:stage=0 layers=3 -[default0]: 0: _to_float16 -[default0]: 1: EmbeddingPipe -[default0]: 2: -[default0]:stage=1 layers=1 -[default0]: 3: ParallelTransformerLayerPipe -[default0]:stage=2 layers=1 -[default0]: 4: ParallelTransformerLayerPipe -[default0]:stage=3 layers=1 -[default0]: 5: ParallelTransformerLayerPipe -[default0]:stage=4 layers=1 -[default0]: 6: ParallelTransformerLayerPipe -[default0]:stage=5 layers=1 -[default0]: 7: ParallelTransformerLayerPipe -[default0]:stage=6 layers=1 -[default0]: 8: ParallelTransformerLayerPipe -[default0]:stage=7 layers=1 -[default0]: 9: ParallelTransformerLayerPipe -[default0]:stage=8 layers=1 -[default0]: 10: ParallelTransformerLayerPipe -[default0]:stage=9 layers=1 -[default0]: 11: ParallelTransformerLayerPipe -[default0]:stage=10 layers=1 -[default0]: 12: ParallelTransformerLayerPipe -[default0]:stage=11 layers=1 -[default0]: 13: ParallelTransformerLayerPipe -[default0]:stage=12 layers=1 -[default0]: 14: ParallelTransformerLayerPipe -[default0]:stage=13 layers=1 -[default0]: 15: ParallelTransformerLayerPipe -[default0]:stage=14 layers=1 -[default0]: 16: ParallelTransformerLayerPipe -[default0]:stage=15 layers=1 -[default0]: 17: ParallelTransformerLayerPipe -[default0]:stage=16 layers=1 -[default0]: 18: ParallelTransformerLayerPipe -[default0]:stage=17 layers=1 -[default0]: 19: ParallelTransformerLayerPipe -[default0]:stage=18 layers=1 -[default0]: 20: ParallelTransformerLayerPipe -[default0]:stage=19 layers=1 -[default0]: 21: ParallelTransformerLayerPipe -[default0]:stage=20 layers=1 -[default0]: 22: ParallelTransformerLayerPipe -[default0]:stage=21 layers=1 -[default0]: 23: ParallelTransformerLayerPipe -[default0]:stage=22 layers=1 -[default0]: 24: ParallelTransformerLayerPipe -[default0]:stage=23 layers=1 -[default0]: 25: ParallelTransformerLayerPipe -[default0]:stage=24 layers=1 -[default0]: 26: ParallelTransformerLayerPipe -[default0]:stage=25 layers=1 -[default0]: 27: ParallelTransformerLayerPipe -[default0]:stage=26 layers=1 -[default0]: 28: ParallelTransformerLayerPipe -[default0]:stage=27 layers=1 -[default0]: 29: ParallelTransformerLayerPipe -[default0]:stage=28 layers=1 -[default0]: 30: ParallelTransformerLayerPipe -[default0]:stage=29 layers=1 -[default0]: 31: ParallelTransformerLayerPipe -[default0]:stage=30 layers=1 -[default0]: 32: ParallelTransformerLayerPipe -[default0]:stage=31 layers=1 -[default0]: 33: ParallelTransformerLayerPipe -[default0]:stage=32 layers=1 -[default0]: 34: ParallelTransformerLayerPipe -[default0]:stage=33 layers=1 -[default0]: 35: ParallelTransformerLayerPipe -[default0]:stage=34 layers=1 -[default0]: 36: ParallelTransformerLayerPipe -[default0]:stage=35 layers=1 -[default0]: 37: ParallelTransformerLayerPipe -[default0]:stage=36 layers=1 -[default0]: 38: ParallelTransformerLayerPipe -[default0]:stage=37 layers=1 -[default0]: 39: ParallelTransformerLayerPipe -[default0]:stage=38 layers=1 -[default0]: 40: ParallelTransformerLayerPipe -[default0]:stage=39 layers=1 -[default0]: 41: ParallelTransformerLayerPipe -[default0]:stage=40 layers=1 -[default0]: 42: ParallelTransformerLayerPipe -[default0]:stage=41 layers=1 -[default0]: 43: ParallelTransformerLayerPipe -[default0]:stage=42 layers=1 -[default0]: 44: ParallelTransformerLayerPipe -[default0]:stage=43 layers=1 -[default0]: 45: ParallelTransformerLayerPipe -[default0]:stage=44 layers=1 -[default0]: 46: ParallelTransformerLayerPipe -[default0]:stage=45 layers=1 -[default0]: 47: ParallelTransformerLayerPipe -[default0]:stage=46 layers=1 -[default0]: 48: ParallelTransformerLayerPipe -[default0]:stage=47 layers=1 -[default0]: 49: ParallelTransformerLayerPipe -[default0]:stage=48 layers=1 -[default0]: 50: ParallelTransformerLayerPipe -[default0]:stage=49 layers=1 -[default0]: 51: ParallelTransformerLayerPipe -[default0]:stage=50 layers=1 -[default0]: 52: ParallelTransformerLayerPipe -[default0]:stage=51 layers=1 -[default0]: 53: ParallelTransformerLayerPipe -[default0]:stage=52 layers=1 -[default0]: 54: ParallelTransformerLayerPipe -[default0]:stage=53 layers=1 -[default0]: 55: ParallelTransformerLayerPipe -[default0]:stage=54 layers=1 -[default0]: 56: ParallelTransformerLayerPipe -[default0]:stage=55 layers=1 -[default0]: 57: ParallelTransformerLayerPipe -[default0]:stage=56 layers=1 -[default0]: 58: ParallelTransformerLayerPipe -[default0]:stage=57 layers=1 -[default0]: 59: ParallelTransformerLayerPipe -[default0]:stage=58 layers=1 -[default0]: 60: ParallelTransformerLayerPipe -[default0]:stage=59 layers=1 -[default0]: 61: ParallelTransformerLayerPipe -[default0]:stage=60 layers=1 -[default0]: 62: ParallelTransformerLayerPipe -[default0]:stage=61 layers=1 -[default0]: 63: ParallelTransformerLayerPipe -[default0]:stage=62 layers=1 -[default0]: 64: ParallelTransformerLayerPipe -[default0]:stage=63 layers=1 -[default0]: 65: ParallelTransformerLayerPipe -[default0]:stage=64 layers=1 -[default0]: 66: ParallelTransformerLayerPipe -[default0]:stage=65 layers=1 -[default0]: 67: ParallelTransformerLayerPipe -[default0]:stage=66 layers=1 -[default0]: 68: ParallelTransformerLayerPipe -[default0]:stage=67 layers=1 -[default0]: 69: ParallelTransformerLayerPipe -[default0]:stage=68 layers=1 -[default0]: 70: ParallelTransformerLayerPipe -[default0]:stage=69 layers=1 -[default0]: 71: ParallelTransformerLayerPipe -[default0]:stage=70 layers=3 -[default0]: 72: ParallelTransformerLayerPipe -[default0]: 73: undo -[default0]: 74: MixedFusedLayerNorm -[default0]:stage=71 layers=2 -[default0]: 75: EmbeddingPipe -[default0]: 76: float16_to_fp32 -[default0]: loss: CrossEntropy -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:[2022-09-07 22:14:33,525] [INFO] [utils.py:827:see_memory_usage] After Building Model -[default0]:[2022-09-07 22:14:33,526] [INFO] [utils.py:828:see_memory_usage] MA 6.7 GB Max_MA 6.7 GB CA 6.7 GB Max_CA 7 GB -[default0]:[2022-09-07 22:14:33,526] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.02 GB, percent = 6.4% -[default0]:setting training iterations to 3100 -[default0]:> learning rate decay style: constant -[default0]:DeepSpeed is enabled. -[default0]:[2022-09-07 22:14:33,527] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.1+8b2a6371, git-hash=8b2a6371, git-branch=master -[default6]:Emitting ninja build file /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113/utils/build.ninja... -[default6]:Building extension module utils... -[default6]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default6]:ninja: no work to do. -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.6954257488250732 seconds -[default0]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6952614784240723 seconds -[default0]:Time to load utils op: 0.6951496601104736 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6952502727508545 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.7470479011535645 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.7470240592956543 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.7470266819000244 seconds -[default0]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.7470285892486572 seconds -[default0]:Loading extension module utils... -[default1]:Loading extension module utils... -[default2]:Loading extension module utils... -[default1]:Loading extension module utils... -[default6]:Loading extension module utils... -[default7]:Loading extension module utils... -[default4]:Loading extension module utils... -[default2]:Loading extension module utils... -[default0]:Loading extension module utils... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6824216842651367 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6835808753967285 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6360819339752197 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6352396011352539 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6353354454040527 seconds -[default1]:Loading extension module utils... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6825003623962402 seconds -[default3]:Loading extension module utils... -[default1]:Loading extension module utils... -[default3]:Time to load utils op: 0.6040685176849365 seconds -[default1]:Time to load utils op: 0.6039605140686035 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6039028167724609 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6353244781494141 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6040544509887695 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.7257678508758545 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6824531555175781 seconds -[default6]:Loading extension module utils... -[default3]:Loading extension module utils... -[default4]:Loading extension module utils... -[default1]:Loading extension module utils... -[default5]:Loading extension module utils... -[default3]:Time to load utils op: 0.7767877578735352 seconds -[default7]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.7767741680145264 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6972527503967285 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.6967604160308838 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6038296222686768 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6039218902587891 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6037144660949707 seconds -[default3]:Loading extension module utils... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6039531230926514 seconds -[default0]:Loading extension module utils... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.6103086471557617 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6825861930847168 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.6517806053161621 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6520195007324219 seconds -[default4]:Loading extension module utils... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6742839813232422 seconds -[default4]:Time to load utils op: 0.6748955249786377 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6819117069244385 seconds -[default5]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6829004287719727 seconds -[default5]:Time to load utils op: 0.6743731498718262 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6742503643035889 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6817033290863037 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6511564254760742 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6835503578186035 seconds -[default1]:Loading extension module utils... -[default4]:Loading extension module utils... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6826701164245605 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.682612419128418 seconds -[default3]:Loading extension module utils... -[default5]:Loading extension module utils... -[default6]:Loading extension module utils... -[default0]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6962790489196777 seconds -[default0]:Time to load utils op: 0.607001543045044 seconds -[default2]:Loading extension module utils... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.6359434127807617 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.7434670925140381 seconds -[default6]:Loading extension module utils... -[default3]:Loading extension module utils... -[default7]:Loading extension module utils... -[default1]:Loading extension module utils... -[default2]:Loading extension module utils... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6826977729797363 seconds -[default4]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.7434885501861572 seconds -[default5]:Loading extension module utils... -[default0]:Loading extension module utils... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.754502534866333 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6361672878265381 seconds -[default1]:Loading extension module utils... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.7434959411621094 seconds -[default1]:Time to load utils op: 0.7434885501861572 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.69083571434021 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6098024845123291 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6049940586090088 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6894934177398682 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6893670558929443 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6961097717285156 seconds -[default0]:Loading extension module utils... -[default1]:Loading extension module utils... -[default2]:Loading extension module utils... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6218314170837402 seconds -[default3]:Loading extension module utils... -[default2]:Loading extension module utils... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6358766555786133 seconds -[default2]:Time to load utils op: 0.6359837055206299 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6898708343505859 seconds -[default4]:Loading extension module utils... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6864399909973145 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.754483699798584 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.70143723487854 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.7162783145904541 seconds -[default5]:Loading extension module utils... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.716355562210083 seconds -[default5]:Time to load utils op: 0.71640944480896 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.716280460357666 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.6858913898468018 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6856222152709961 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6390960216522217 seconds -[default2]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6953208446502686 seconds -[default2]:Time to load utils op: 0.6857526302337646 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6215360164642334 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6215982437133789 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.621711015701294 seconds -[default5]:Loading extension module utils... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.8083047866821289 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.8082823753356934 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6971077919006348 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.701549768447876 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6318190097808838 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.7018797397613525 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6316866874694824 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.688720703125 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.7020847797393799 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.638481616973877 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.7392477989196777 seconds -[default1]:Loading extension module utils... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.7159554958343506 seconds -[default1]:Time to load utils op: 0.692652702331543 seconds -[default3]:Loading extension module utils... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.7391514778137207 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.8082962036132812 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.7399301528930664 seconds -[default3]:Time to load utils op: 0.8083090782165527 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6953439712524414 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6953446865081787 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6971054077148438 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6953320503234863 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6926224231719971 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.697105884552002 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.7542767524719238 seconds -[default1]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6827759742736816 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6714134216308594 seconds -[default1]:Time to load utils op: 0.6709389686584473 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6706972122192383 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.68353271484375 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6543848514556885 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6835274696350098 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6835355758666992 seconds -[default2]:Loading extension module utils... -[default0]:Loading extension module utils... -[default2]:Time to load utils op: 1.1963603496551514 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.196242094039917 seconds -[default0]:Time to load utils op: 1.1962478160858154 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6511101722717285 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.7153000831604004 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.7153472900390625 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.688366174697876 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6706163883209229 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.7057821750640869 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.716684103012085 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.6807982921600342 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6879231929779053 seconds -[default4]:Loading extension module utils... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6618752479553223 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.7057905197143555 seconds -[default4]:Time to load utils op: 0.6644899845123291 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6618046760559082 seconds -[default1]:Loading extension module utils... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6758055686950684 seconds -[default1]:Time to load utils op: 0.6615016460418701 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6856622695922852 seconds -[default6]:Loading extension module utils... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.686530590057373 seconds -[default6]:Time to load utils op: 0.6649231910705566 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6815943717956543 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.7499854564666748 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.7499909400939941 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6873116493225098 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.7545144557952881 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6762068271636963 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6856734752655029 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.6873056888580322 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6648187637329102 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6543912887573242 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6811704635620117 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6804189682006836 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6803765296936035 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6828176975250244 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6832871437072754 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6830217838287354 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.1962389945983887 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.705775260925293 seconds -[default1]:Loading extension module utils... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.7333691120147705 seconds -[default1]:Time to load utils op: 0.7331347465515137 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6420893669128418 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.6543798446655273 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6427583694458008 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6425909996032715 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.6543693542480469 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.7331507205963135 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.7057840824127197 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6674094200134277 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.667954683303833 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6665911674499512 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.7331459522247314 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.64212965965271 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.6624150276184082 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6644282341003418 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6870236396789551 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.750016450881958 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.7500190734863281 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6857352256774902 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.6859791278839111 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6646053791046143 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.7392597198486328 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6667213439941406 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.6812090873718262 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.6932311058044434 seconds -[default7]:Loading extension module utils... -[default6]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.6932141780853271 seconds -[default6]:Time to load utils op: 0.6932120323181152 seconds -[default7]:Time to load utils op: 0.6931955814361572 seconds -[default3]:Loading extension module utils... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.6807563304901123 seconds -[default3]:Time to load utils op: 0.6809170246124268 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.7084362506866455 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.6644318103790283 seconds -[default7]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.2065551280975342 seconds -[default7]:Time to load utils op: 1.2052874565124512 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.2052562236785889 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.2050879001617432 seconds -[default0]:Time to load utils op: 0.7264680862426758 seconds -[default1]:Time to load utils op: 0.7832276821136475 seconds -[default0]:Time to load utils op: 0.783245325088501 seconds -[default1]:Time to load utils op: 0.7257606983184814 seconds -[default2]:Time to load utils op: 0.7262547016143799 seconds -[default6]:Time to load utils op: 0.7207295894622803 seconds -[default7]:Time to load utils op: 0.7210869789123535 seconds -[default4]:Time to load utils op: 0.7215325832366943 seconds -[default2]:Time to load utils op: 0.7832379341125488 seconds -[default0]:Time to load utils op: 0.776775598526001 seconds -[default1]:Time to load utils op: 0.6824440956115723 seconds -[default6]:Time to load utils op: 0.7130513191223145 seconds -[default4]:Time to load utils op: 0.7143106460571289 seconds -[default5]:Time to load utils op: 0.7136721611022949 seconds -[default1]:Time to load utils op: 0.7771909236907959 seconds -[default7]:Time to load utils op: 0.7129693031311035 seconds -[default3]:Time to load utils op: 0.7832298278808594 seconds -[default1]:Time to load utils op: 0.666351318359375 seconds -[default4]:Time to load utils op: 0.6879189014434814 seconds -[default3]:Time to load utils op: 0.6662335395812988 seconds -[default5]:Time to load utils op: 0.7208967208862305 seconds -[default0]:Time to load utils op: 0.6671104431152344 seconds -[default6]:Time to load utils op: 0.6879353523254395 seconds -[default2]:Time to load utils op: 0.6662237644195557 seconds -[default3]:Time to load utils op: 0.6812934875488281 seconds -[default6]:Time to load utils op: 0.6744225025177002 seconds -[default7]:Time to load utils op: 0.6742980480194092 seconds -[default1]:Time to load utils op: 0.681307315826416 seconds -[default2]:Time to load utils op: 0.6812894344329834 seconds -[default4]:Time to load utils op: 0.6751041412353516 seconds -[default5]:Time to load utils op: 0.6749374866485596 seconds -[default0]:Time to load utils op: 0.6813197135925293 seconds -[default0]:Time to load utils op: 0.6770782470703125 seconds -[default1]:Time to load utils op: 0.6769511699676514 seconds -[default2]:Time to load utils op: 0.676429271697998 seconds -[default3]:Time to load utils op: 0.6763522624969482 seconds -[default4]:Time to load utils op: 0.6970901489257812 seconds -[default5]:Time to load utils op: 0.6879823207855225 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.005579233169555664 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0017235279083251953 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0014910697937011719 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0014638900756835938 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0013279914855957031 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0012736320495605469 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.001428365707397461 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.001468658447265625 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.001371145248413086 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0006177425384521484 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0007977485656738281 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006511211395263672 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005242824554443359 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.00063323974609375 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.000690460205078125 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0004935264587402344 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0008745193481445312 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006933212280273438 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006868839263916016 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006353855133056641 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005905628204345703 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005543231964111328 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0010101795196533203 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0008769035339355469 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007903575897216797 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0011734962463378906 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0012629032135009766 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.00152587890625 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006079673767089844 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006341934204101562 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005524158477783203 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0012946128845214844 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006413459777832031 seconds -[default2]:Time to load utils op: 0.0005230903625488281 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005795955657958984 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0007603168487548828 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0007348060607910156 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006563663482666016 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005867481231689453 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0009031295776367188 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006992816925048828 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006742477416992188 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0006012916564941406 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005943775177001953 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0008306503295898438 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006496906280517578 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006718635559082031 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006072521209716797 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0008144378662109375 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005483627319335938 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.000583648681640625 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0015230178833007812 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0016324520111083984 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0004987716674804688 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0008480548858642578 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005548000335693359 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005304813385009766 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0007622241973876953 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007245540618896484 seconds -[default7]:Time to load utils op: 0.0006170272827148438 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0007212162017822266 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006301403045654297 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006887912750244141 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006349086761474609 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0007998943328857422 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0004811286926269531 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0004889965057373047 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.000804901123046875 seconds -[default3]:Time to load utils op: 0.0006871223449707031 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.000614166259765625 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006456375122070312 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005459785461425781 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0004820823669433594 seconds -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default0]:Time to load utils op: 0.0004642009735107422 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0008711814880371094 seconds -[default2]:Time to load utils op: 0.0004277229309082031 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0006077289581298828 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005428791046142578 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006411075592041016 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005640983581542969 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005233287811279297 seconds -[default6]:Emitting ninja build file /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113/utils/build.ninja... -[default6]:Building extension module utils... -[default6]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006778240203857422 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0008168220520019531 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0008325576782226562 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0007770061492919922 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005640983581542969 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005207061767578125 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0007555484771728516 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0011491775512695312 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005197525024414062 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0007226467132568359 seconds -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007879734039306641 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0015559196472167969 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.001672506332397461 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006709098815917969 seconds -[default6]:Time to load utils op: 0.000598907470703125 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006659030914306641 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0011014938354492188 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0007529258728027344 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005614757537841797 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005822181701660156 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005943775177001953 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0010755062103271484 seconds -[default3]:Time to load utils op: 0.0008497238159179688 seconds -[default5]:Time to load utils op: 0.0005929470062255859 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005645751953125 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0007116794586181641 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0007474422454833984 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0008120536804199219 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005140304565429688 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006144046783447266 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005414485931396484 seconds -[default7]:Time to load utils op: 0.0006442070007324219 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005054473876953125 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005776882171630859 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0011222362518310547 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0004825592041015625 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0014033317565917969 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006167888641357422 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0015833377838134766 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005290508270263672 seconds -[default5]:Time to load utils op: 0.0013816356658935547 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0006206035614013672 seconds -[default1]:Time to load utils op: 0.0005435943603515625 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0007467269897460938 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0007278919219970703 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0018563270568847656 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006515979766845703 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0013294219970703125 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.000759124755859375 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005691051483154297 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005948543548583984 seconds -[default6]:Time to load utils op: 0.0018367767333984375 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0010232925415039062 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0007719993591308594 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005502700805664062 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0007171630859375 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005195140838623047 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.00054168701171875 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005965232849121094 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0007178783416748047 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006198883056640625 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005805492401123047 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006246566772460938 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.000789642333984375 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0007767677307128906 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006208419799804688 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.000606536865234375 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005495548248291016 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005230903625488281 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005955696105957031 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0015599727630615234 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006666183471679688 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006620883941650391 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0007421970367431641 seconds -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0003502368927001953 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0006566047668457031 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0007233619689941406 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005078315734863281 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006470680236816406 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0013458728790283203 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005717277526855469 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006964206695556641 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005609989166259766 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.00048661231994628906 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.00046133995056152344 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007345676422119141 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005338191986083984 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0003521442413330078 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006499290466308594 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005211830139160156 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0015497207641601562 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0010547637939453125 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006937980651855469 seconds -[default4]:Time to load utils op: 0.0006685256958007812 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005404949188232422 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.000453948974609375 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005295276641845703 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005395412445068359 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0007450580596923828 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.00046944618225097656 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0016047954559326172 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005619525909423828 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0010082721710205078 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0014629364013671875 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005640983581542969 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006320476531982422 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.00048828125 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006308555603027344 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005297660827636719 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005538463592529297 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.00044155120849609375 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005350112915039062 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005280971527099609 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006144046783447266 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0015404224395751953 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0017600059509277344 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0012619495391845703 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.001344442367553711 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0014824867248535156 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0007891654968261719 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0007433891296386719 seconds -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005273818969726562 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0008401870727539062 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006892681121826172 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005364418029785156 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0013430118560791016 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005972385406494141 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006361007690429688 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005593299865722656 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0007503032684326172 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005636215209960938 seconds -[default6]:Time to load utils op: 0.0007503032684326172 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006356239318847656 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006303787231445312 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005245208740234375 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006008148193359375 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005865097045898438 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0005140304565429688 seconds -[default5]:Time to load utils op: 0.0007002353668212891 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007314682006835938 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default6]:Time to load utils op: 0.0007512569427490234 seconds -[default3]:Time to load utils op: 0.0005393028259277344 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006899833679199219 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006575584411621094 seconds -[default6]:ninja: no work to do. -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.9394862651824951 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:[2022-09-07 22:14:34,304] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False -[default0]:[2022-09-07 22:14:34,305] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer -[default0]:[2022-09-07 22:14:34,305] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer -[default0]:[2022-09-07 22:14:34,305] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = {basic_optimizer.__class__.__name__} -[default0]:[2022-09-07 22:14:34,305] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer -[default0]:[2022-09-07 22:14:34,351] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0015988349914550781 seconds -[default0]:[2022-09-07 22:14:34,351] [INFO] [utils.py:828:see_memory_usage] MA 6.7 GB Max_MA 6.7 GB CA 6.7 GB Max_CA 7 GB -[default0]:[2022-09-07 22:14:34,352] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.2 GB, percent = 6.4% -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Loading extension module utils... -[default3]:Emitting ninja build file /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113/utils/build.ninja... -[default3]:Building extension module utils... -[default3]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) -[default3]:ninja: no work to do. -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.23862862586975098 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.2680823802947998 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.258387804031372 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.2582838535308838 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.2594244480133057 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.2683274745941162 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.258270263671875 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.2945940494537354 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.2679224014282227 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.257296085357666 seconds -[default2]:Loading extension module utils... -[default1]:Loading extension module utils... -[default2]:Time to load utils op: 1.2945876121520996 seconds -[default1]:Time to load utils op: 1.2945902347564697 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.2431514263153076 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.2947089672088623 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.2947235107421875 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.2477984428405762 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.2478156089782715 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.2478423118591309 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.2294533252716064 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.2426414489746094 seconds -[default4]:Loading extension module utils... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.2478437423706055 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.2290797233581543 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.2419745922088623 seconds -[default4]:Time to load utils op: 1.229825496673584 seconds -[default3]:Loading extension module utils... -[default6]:Loading extension module utils... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.2293121814727783 seconds -[default6]:Time to load utils op: 1.2417538166046143 seconds -[default3]:Time to load utils op: 1.2945563793182373 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.2947063446044922 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.2424981594085693 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.2419922351837158 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.2572882175445557 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.2572903633117676 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.2572932243347168 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.2420129776000977 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.2154788970947266 seconds -[default1]:Loading extension module utils... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.3091604709625244 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.32019782066345215 seconds -[default1]:Time to load utils op: 1.287320852279663 seconds -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 1.286726951599121 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.286741018295288 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.2880284786224365 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.309168815612793 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.214101791381836 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.214005470275879 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.214322566986084 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.3091473579406738 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.32027292251586914 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 1.2232646942138672 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.2820346355438232 seconds -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.2025163173675537 seconds -[default0]:[2022-09-07 22:14:34,586] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 -[default0]:[2022-09-07 22:14:34,586] [INFO] [utils.py:828:see_memory_usage] MA 6.7 GB Max_MA 6.7 GB CA 6.7 GB Max_CA 7 GB -[default0]:[2022-09-07 22:14:34,587] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.25 GB, percent = 6.4% -[default0]:[2022-09-07 22:14:34,645] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 -[default0]:[2022-09-07 22:14:34,645] [INFO] [utils.py:828:see_memory_usage] MA 23.45 GB Max_MA 23.45 GB CA 25.12 GB Max_CA 25 GB -[default0]:[2022-09-07 22:14:34,645] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.25 GB, percent = 6.4% -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 1.3091623783111572 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 1.2429404258728027 seconds -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.31974101066589355 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 1.2222213745117188 seconds -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.30806517601013184 seconds -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 1.2815866470336914 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 1.2818920612335205 seconds -[default3]:Loading extension module utils... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 1.2226247787475586 seconds -[default3]:Time to load utils op: 1.2222363948822021 seconds -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.31975531578063965 seconds -[default0]:Time to load utils op: 1.2947068214416504 seconds -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.3080620765686035 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.000514984130859375 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.002222776412963867 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006234645843505859 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006566047668457031 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007214546203613281 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006220340728759766 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0009100437164306641 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005571842193603516 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0007328987121582031 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006327629089355469 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005946159362792969 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006690025329589844 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0007824897766113281 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005409717559814453 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0006883144378662109 seconds -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.00064849853515625 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005698204040527344 seconds -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006241798400878906 seconds -[default6]:Time to load utils op: 0.0007851123809814453 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0007557868957519531 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0007836818695068359 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006442070007324219 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0018210411071777344 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0016758441925048828 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0017876625061035156 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0006866455078125 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.001905679702758789 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005495548248291016 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0021753311157226562 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.001760721206665039 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.001737833023071289 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0006647109985351562 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0008165836334228516 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0007321834564208984 seconds -[default0]:[2022-09-07 22:14:34,673] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 -[default0]:[2022-09-07 22:14:34,673] [INFO] [utils.py:828:see_memory_usage] MA 23.45 GB Max_MA 23.45 GB CA 25.12 GB Max_CA 25 GB -[default0]:[2022-09-07 22:14:34,673] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.25 GB, percent = 6.4% -[default0]:[2022-09-07 22:14:34,702] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 -[default0]:[2022-09-07 22:14:34,702] [INFO] [utils.py:828:see_memory_usage] MA 23.45 GB Max_MA 23.45 GB CA 25.12 GB Max_CA 25 GB -[default0]:[2022-09-07 22:14:34,703] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.25 GB, percent = 6.4% -[default0]:[2022-09-07 22:14:34,730] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer -[default0]:[2022-09-07 22:14:34,730] [INFO] [utils.py:828:see_memory_usage] MA 23.45 GB Max_MA 23.45 GB CA 25.12 GB Max_CA 25 GB -[default0]:[2022-09-07 22:14:34,731] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.25 GB, percent = 6.4% -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0006177425384521484 seconds -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005316734313964844 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0006868839263916016 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0003418922424316406 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0008971691131591797 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0007345676422119141 seconds -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default1]:Time to load utils op: 0.0006692409515380859 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:Time to load utils op: 0.0007092952728271484 seconds -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0008261203765869141 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0008149147033691406 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0003170967102050781 seconds -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0007052421569824219 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005478858947753906 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005340576171875 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default1]:Time to load utils op: 0.0005137920379638672 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.0005047321319580078 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.000583648681640625 seconds -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0006849765777587891 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0007576942443847656 seconds -[default4]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default4]:No modifications detected for re-loaded extension module utils, skipping build step... -[default4]:Loading extension module utils... -[default4]:Time to load utils op: 0.000713348388671875 seconds -[default3]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default3]:No modifications detected for re-loaded extension module utils, skipping build step... -[default2]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default2]:No modifications detected for re-loaded extension module utils, skipping build step... -[default3]:Loading extension module utils... -[default3]:Time to load utils op: 0.0005857944488525391 seconds -[default1]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default1]:No modifications detected for re-loaded extension module utils, skipping build step... -[default1]:Loading extension module utils... -[default2]:Loading extension module utils... -[default2]:Time to load utils op: 0.0005748271942138672 seconds -[default1]:Time to load utils op: 0.0006470680236816406 seconds -[default5]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.0005288124084472656 seconds -[default5]:No modifications detected for re-loaded extension module utils, skipping build step... -[default5]:Loading extension module utils... -[default5]:Time to load utils op: 0.0005893707275390625 seconds -[default7]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default7]:No modifications detected for re-loaded extension module utils, skipping build step... -[default7]:Loading extension module utils... -[default7]:Time to load utils op: 0.0006101131439208984 seconds -[default0]:[2022-09-07 22:14:34,795] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer -[default0]:[2022-09-07 22:14:34,795] [INFO] [utils.py:828:see_memory_usage] MA 30.15 GB Max_MA 30.15 GB CA 31.82 GB Max_CA 32 GB -[default0]:[2022-09-07 22:14:34,795] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.25 GB, percent = 6.4% -[default0]:[2022-09-07 22:14:34,824] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer -[default0]:[2022-09-07 22:14:34,824] [INFO] [utils.py:828:see_memory_usage] MA 30.15 GB Max_MA 30.15 GB CA 31.82 GB Max_CA 32 GB -[default0]:[2022-09-07 22:14:34,824] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.26 GB, percent = 6.4% -[default0]:[2022-09-07 22:14:34,825] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam -[default0]:[2022-09-07 22:14:34,825] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler -[default0]:[2022-09-07 22:14:34,825] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = -[default0]:[2022-09-07 22:14:34,825] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[2e-05, 2e-05], mom=[(0.9, 0.95), (0.9, 0.95)] -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:987:print] DeepSpeedEngine configuration: -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] activation_checkpointing_config { -[default0]: "partition_activations": false, -[default0]: "contiguous_memory_optimization": false, -[default0]: "cpu_checkpointing": false, -[default0]: "number_checkpoints": null, -[default0]: "synchronize_checkpoint_boundary": false, -[default0]: "profile": false -[default0]:} -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] amp_enabled .................. False -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] amp_params ................... False -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] autotuning_config ............ { -[default0]: "enabled": false, -[default0]: "start_step": null, -[default0]: "end_step": null, -[default0]: "metric_path": null, -[default0]: "arg_mappings": null, -[default0]: "metric": "throughput", -[default0]: "model_info": null, -[default0]: "results_dir": null, -[default0]: "exps_dir": null, -[default0]: "overwrite": true, -[default0]: "fast": true, -[default0]: "start_profile_step": 3, -[default0]: "end_profile_step": 5, -[default0]: "tuner_type": "gridsearch", -[default0]: "tuner_early_stopping": 5, -[default0]: "tuner_num_trials": 50, -[default0]: "model_info_path": null, -[default0]: "mp_size": 1, -[default0]: "max_train_batch_size": null, -[default0]: "min_train_batch_size": 1, -[default0]: "max_train_micro_batch_size_per_gpu": 1.024000e+03, -[default0]: "min_train_micro_batch_size_per_gpu": 1, -[default0]: "num_tuning_micro_batch_sizes": 3 -[default0]:} -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] bfloat16_enabled ............. True -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] checkpoint_tag_validation_enabled True -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] checkpoint_tag_validation_fail False -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] comms_config ................. -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] communication_data_type ...... None -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} -[default0]:[2022-09-07 22:14:34,825] [INFO] [config.py:991:print] curriculum_enabled ........... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] curriculum_params ............ False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] dataloader_drop_last ......... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] disable_allgather ............ False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] dump_state ................... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] dynamic_loss_scale_args ...... None -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] eigenvalue_enabled ........... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] eigenvalue_gas_boundary_resolution 1 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] eigenvalue_layer_name ........ bert.encoder.layer -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] eigenvalue_layer_num ......... 0 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] eigenvalue_max_iter .......... 100 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] eigenvalue_stability ......... 1e-06 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] eigenvalue_tol ............... 0.01 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] eigenvalue_verbose ........... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] elasticity_enabled ........... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] flops_profiler_config ........ { -[default0]: "enabled": false, -[default0]: "profile_step": 1, -[default0]: "module_depth": -1, -[default0]: "top_modules": 1, -[default0]: "detailed": true, -[default0]: "output_file": null -[default0]:} -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] fp16_auto_cast ............... None -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] fp16_enabled ................. False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] fp16_master_weights_and_gradients False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] global_rank .................. 0 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] gradient_accumulation_steps .. 512 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] gradient_clipping ............ 1.0 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] gradient_predivide_factor .... 1.0 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] initial_dynamic_scale ........ 1 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] load_universal_checkpoint .... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] loss_scale ................... 1.0 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] memory_breakdown ............. False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] monitor_config ............... -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] nebula_config ................ { -[default0]: "enabled": false, -[default0]: "persistent_storage_path": null, -[default0]: "persistent_time_interval": 100, -[default0]: "num_of_version_in_retention": 2, -[default0]: "enable_nebula_load": true, -[default0]: "load_path": null -[default0]:} -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] optimizer_legacy_fusion ...... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] optimizer_name ............... None -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] optimizer_params ............. None -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] pld_enabled .................. False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] pld_params ................... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] prescale_gradients ........... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] scheduler_name ............... None -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] scheduler_params ............. None -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] sparse_attention ............. None -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] sparse_gradients_enabled ..... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] steps_per_print .............. 2000 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] train_batch_size ............. 2048 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] train_micro_batch_size_per_gpu 1 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] wall_clock_breakdown ......... False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] world_size ................... 4 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] zero_allow_untested_optimizer False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] zero_enabled ................. False -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:991:print] zero_optimization_stage ...... 0 -[default0]:[2022-09-07 22:14:34,826] [INFO] [config.py:976:print_user_config] json = { -[default0]: "train_micro_batch_size_per_gpu": 1, -[default0]: "train_batch_size": 2.048000e+03, -[default0]: "gradient_clipping": 1.0, -[default0]: "zero_optimization": { -[default0]: "stage": 0 -[default0]: }, -[default0]: "bf16": { -[default0]: "enabled": true -[default0]: }, -[default0]: "steps_per_print": 2.000000e+03, -[default0]: "wall_clock_breakdown": false -[default0]:} -[default0]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default0]:No modifications detected for re-loaded extension module utils, skipping build step... -[default0]:Loading extension module utils... -[default0]:Time to load utils op: 0.00047898292541503906 seconds -[default0]:[2022-09-07 22:14:34,827] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=512 micro_batch_size=1 -[default6]:Using /gpfs7kw/linkhome/rech/genhug01/unj46ad/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... -[default6]:No modifications detected for re-loaded extension module utils, skipping build step... -[default6]:Loading extension module utils... -[default6]:Time to load utils op: 0.0005831718444824219 seconds -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=40 STAGE=10 LAYERS=1 [12, 13) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=272 STAGE=68 LAYERS=1 [70, 71) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=112 STAGE=28 LAYERS=1 [30, 31) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=200 STAGE=50 LAYERS=1 [52, 53) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=168 STAGE=42 LAYERS=1 [44, 45) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=172 STAGE=43 LAYERS=1 [45, 46) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=204 STAGE=51 LAYERS=1 [53, 54) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=264 STAGE=66 LAYERS=1 [68, 69) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=276 STAGE=69 LAYERS=1 [71, 72) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=44 STAGE=11 LAYERS=1 [13, 14) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=144 STAGE=36 LAYERS=1 [38, 39) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=268 STAGE=67 LAYERS=1 [69, 70) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=120 STAGE=30 LAYERS=1 [32, 33) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=148 STAGE=37 LAYERS=1 [39, 40) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=192 STAGE=48 LAYERS=1 [50, 51) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=56 STAGE=14 LAYERS=1 [16, 17) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=116 STAGE=29 LAYERS=1 [31, 32) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=60 STAGE=15 LAYERS=1 [17, 18) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=72 STAGE=18 LAYERS=1 [20, 21) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=32 STAGE=8 LAYERS=1 [10, 11) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=228 STAGE=57 LAYERS=1 [59, 60) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=224 STAGE=56 LAYERS=1 [58, 59) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=8 STAGE=2 LAYERS=1 [4, 5) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=20 STAGE=5 LAYERS=1 [7, 8) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=16 STAGE=4 LAYERS=1 [6, 7) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=96 STAGE=24 LAYERS=1 [26, 27) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=12 STAGE=3 LAYERS=1 [5, 6) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=24 STAGE=6 LAYERS=1 [8, 9) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=128 STAGE=32 LAYERS=1 [34, 35) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=132 STAGE=33 LAYERS=1 [35, 36) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=64 STAGE=16 LAYERS=1 [18, 19) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=176 STAGE=44 LAYERS=1 [46, 47) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=124 STAGE=31 LAYERS=1 [33, 34) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=68 STAGE=17 LAYERS=1 [19, 20) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=28 STAGE=7 LAYERS=1 [9, 10) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=196 STAGE=49 LAYERS=1 [51, 52) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=76 STAGE=19 LAYERS=1 [21, 22) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=256 STAGE=64 LAYERS=1 [66, 67) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=160 STAGE=40 LAYERS=1 [42, 43) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=260 STAGE=65 LAYERS=1 [67, 68) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=216 STAGE=54 LAYERS=1 [56, 57) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=164 STAGE=41 LAYERS=1 [43, 44) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=100 STAGE=25 LAYERS=1 [27, 28) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=156 STAGE=39 LAYERS=1 [41, 42) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=152 STAGE=38 LAYERS=1 [40, 41) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=36 STAGE=9 LAYERS=1 [11, 12) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=180 STAGE=45 LAYERS=1 [47, 48) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=280 STAGE=70 LAYERS=3 [72, 75) STAGE_PARAMS=2466465792 (2466.466M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=140 STAGE=35 LAYERS=1 [37, 38) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=284 STAGE=71 LAYERS=2 [75, 77) STAGE_PARAMS=3596615680 (3596.616M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=136 STAGE=34 LAYERS=1 [36, 37) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=220 STAGE=55 LAYERS=1 [57, 58) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=252 STAGE=63 LAYERS=1 [65, 66) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=52 STAGE=13 LAYERS=1 [15, 16) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=104 STAGE=26 LAYERS=1 [28, 29) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=236 STAGE=59 LAYERS=1 [61, 62) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=232 STAGE=58 LAYERS=1 [60, 61) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=248 STAGE=62 LAYERS=1 [64, 65) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=80 STAGE=20 LAYERS=1 [22, 23) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=184 STAGE=46 LAYERS=1 [48, 49) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=208 STAGE=52 LAYERS=1 [54, 55) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=84 STAGE=21 LAYERS=1 [23, 24) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=212 STAGE=53 LAYERS=1 [55, 56) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=188 STAGE=47 LAYERS=1 [49, 50) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=3 [0, 3) STAGE_PARAMS=3596644352 (3596.644M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=240 STAGE=60 LAYERS=1 [62, 63) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=244 STAGE=61 LAYERS=1 [63, 64) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=48 STAGE=12 LAYERS=1 [14, 15) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,472] [INFO] [engine.py:145:__init__] RANK=108 STAGE=27 LAYERS=1 [29, 30) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=92 STAGE=23 LAYERS=1 [25, 26) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default0]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=88 STAGE=22 LAYERS=1 [24, 25) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default4]:[2022-09-07 22:14:35,471] [INFO] [engine.py:145:__init__] RANK=4 STAGE=1 LAYERS=1 [3, 4) STAGE_PARAMS=2466437120 (2466.437M) TOTAL_PARAMS=179843887104 (179843.887M) UNIQUE_PARAMS=176247271424 (176247.271M) -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_68_model_states.pt... -[default0]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_68_model_states.pt. -[default0]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_68_model_states.pt... -[default1]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_68_model_states.pt. -[default3]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_68_model_states.pt... -[default3]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_68_model_states.pt. -[default3]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_28_model_states.pt... -[default0]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_28_model_states.pt. -[default0]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_68_model_states.pt... -[default2]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_68_model_states.pt. -[default2]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_50_model_states.pt... -[default1]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_50_model_states.pt. -[default1]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_50_model_states.pt... -[default0]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_50_model_states.pt. -[default0]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_51_model_states.pt... -[default4]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_51_model_states.pt. -[default4]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_51_model_states.pt... -[default6]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_51_model_states.pt. -[default6]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_50_model_states.pt... -[default2]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_50_model_states.pt. -[default2]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_28_model_states.pt... -[default1]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_28_model_states.pt. -[default7]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_51_model_states.pt... -[default7]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_51_model_states.pt. -[default7]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_28_model_states.pt... -[default2]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_28_model_states.pt. -[default2]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_69_model_states.pt... -[default7]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_69_model_states.pt. -[default7]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_69_model_states.pt... -[default4]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_69_model_states.pt. -[default4]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_14_model_states.pt... -[default1]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_14_model_states.pt. -[default4]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_11_model_states.pt... -[default4]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_11_model_states.pt... -[default3]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_36_model_states.pt... -[default3]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_36_model_states.pt. -[default7]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_11_model_states.pt... -[default2]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_36_model_states.pt... -[default2]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_36_model_states.pt. -[default2]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_69_model_states.pt... -[default5]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_69_model_states.pt. -[default5]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_11_model_states.pt... -[default0]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_36_model_states.pt... -[default0]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_36_model_states.pt. -[default0]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_36_model_states.pt... -[default1]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_36_model_states.pt. -[default1]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_28_model_states.pt... -[default3]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_28_model_states.pt. -[default3]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_69_model_states.pt... -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_69_model_states.pt. -[default6]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_37_model_states.pt... -[default5]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_37_model_states.pt. -[default5]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_37_model_states.pt... -[default4]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_37_model_states.pt. -[default4]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_37_model_states.pt... -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_37_model_states.pt. -[default6]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_37_model_states.pt... -[default7]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_37_model_states.pt. -[default7]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_14_model_states.pt... -[default0]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_14_model_states.pt. -[default0]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_48_model_states.pt... -[default6]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_48_model_states.pt... -[default0]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_48_model_states.pt. -[default0]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_29_model_states.pt... -[default7]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_29_model_states.pt. -[default2]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_48_model_states.pt. -[default1]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_50_model_states.pt... -[default3]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_50_model_states.pt. -[default3]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_14_model_states.pt... -[default2]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_14_model_states.pt... -[default2]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_14_model_states.pt. -[default7]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_14_model_states.pt. -[default3]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_29_model_states.pt... -[default4]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_29_model_states.pt. -[default4]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_15_model_states.pt... -[default4]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_15_model_states.pt. -[default6]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_29_model_states.pt... -[default6]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_29_model_states.pt. -[default6]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_29_model_states.pt... -[default5]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_29_model_states.pt. -[default4]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_18_model_states.pt... -[default0]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_18_model_states.pt. -[default0]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_08_model_states.pt... -[default1]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_08_model_states.pt. -[default1]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_15_model_states.pt... -[default7]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_15_model_states.pt. -[default7]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_15_model_states.pt... -[default5]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_15_model_states.pt. -[default5]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_51_model_states.pt... -[default5]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_51_model_states.pt. -[default5]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_08_model_states.pt... -[default0]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_08_model_states.pt. -[default0]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_09_model_states.pt... -[default6]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_09_model_states.pt. -[default6]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_02_model_states.pt... -[default0]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_02_model_states.pt. -[default6]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_15_model_states.pt... -[default6]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_15_model_states.pt. -[default6]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_43_model_states.pt... -[default7]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_43_model_states.pt. -[default7]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_43_model_states.pt... -[default6]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_43_model_states.pt. -[default6]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_04_model_states.pt... -[default3]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_04_model_states.pt. -[default3]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_04_model_states.pt... -[default1]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_04_model_states.pt. -[default1]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_05_model_states.pt... -[default7]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_05_model_states.pt. -[default7]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_05_model_states.pt... -[default6]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_05_model_states.pt. -[default6]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_05_model_states.pt... -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_05_model_states.pt... -[default4]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_05_model_states.pt. -[default4]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_05_model_states.pt. -[default0]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_04_model_states.pt... -[default2]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_04_model_states.pt... -[default2]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_04_model_states.pt. -[default5]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_04_model_states.pt. -[default0]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_02_model_states.pt... -[default2]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_02_model_states.pt. -[default2]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_24_model_states.pt... -[default0]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_24_model_states.pt. -[default3]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_02_model_states.pt... -[default4]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_03_model_states.pt... -[default4]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_03_model_states.pt. -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_02_model_states.pt... -[default1]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_02_model_states.pt. -[default3]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_02_model_states.pt. -[default3]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_06_model_states.pt... -[default2]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_48_model_states.pt... -[default2]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_48_model_states.pt. -[default2]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_08_model_states.pt... -[default2]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_08_model_states.pt. -[default2]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_03_model_states.pt... -[default6]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_03_model_states.pt. -[default6]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_48_model_states.pt... -[default3]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_48_model_states.pt. -[default3]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_03_model_states.pt... -[default7]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_03_model_states.pt. -[default7]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_49_model_states.pt... -[default6]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_49_model_states.pt. -[default6]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_06_model_states.pt... -[default3]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_06_model_states.pt. -[default2]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_06_model_states.pt... -[default2]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_06_model_states.pt. -[default5]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_03_model_states.pt... -[default5]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_03_model_states.pt. -[default5]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_24_model_states.pt... -[default2]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_24_model_states.pt. -[default2]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_44_model_states.pt... -[default6]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_25_model_states.pt... -[default6]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_25_model_states.pt. -[default6]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_49_model_states.pt... -[default4]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_49_model_states.pt. -[default4]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_07_model_states.pt... -[default4]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_07_model_states.pt. -[default4]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_49_model_states.pt... -[default7]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_49_model_states.pt. -[default7]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_49_model_states.pt... -[default5]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_49_model_states.pt. -[default5]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_18_model_states.pt... -[default1]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_18_model_states.pt. -[default1]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_18_model_states.pt... -[default2]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_18_model_states.pt. -[default2]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_22_model_states.pt... -[default3]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_22_model_states.pt... -[default4]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_19_model_states.pt... -[default4]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_19_model_states.pt. -[default4]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_19_model_states.pt... -[default5]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_19_model_states.pt. -[default5]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_19_model_states.pt... -[default5]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_25_model_states.pt... -[default5]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_25_model_states.pt. -[default7]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_19_model_states.pt. -[default7]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_40_model_states.pt... -[default0]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_40_model_states.pt. -[default6]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_19_model_states.pt... -[default6]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_19_model_states.pt. -[default5]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_25_model_states.pt... -[default7]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_25_model_states.pt. -[default0]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_18_model_states.pt... -[default3]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_18_model_states.pt. -[default7]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_40_model_states.pt... -[default1]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_40_model_states.pt. -[default3]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_22_model_states.pt... -[default5]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_07_model_states.pt... -[default5]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_07_model_states.pt. -[default5]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_07_model_states.pt... -[default7]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_07_model_states.pt. -[default7]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_07_model_states.pt... -[default6]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_07_model_states.pt. -[default6]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_40_model_states.pt... -[default2]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_40_model_states.pt. -[default2]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_54_model_states.pt... -[default0]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_54_model_states.pt. -[default0]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_23_model_states.pt... -[default5]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_55_model_states.pt... -[default5]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_55_model_states.pt. -[default5]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_54_model_states.pt... -[default1]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_54_model_states.pt. -[default6]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_40_model_states.pt... -[default7]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_41_model_states.pt... -[default7]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_41_model_states.pt. -[default5]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_41_model_states.pt... -[default5]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_41_model_states.pt. -[default4]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_41_model_states.pt... -[default4]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_41_model_states.pt. -[default4]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_40_model_states.pt. -[default3]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_54_model_states.pt... -[default2]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_54_model_states.pt. -[default2]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_24_model_states.pt... -[default3]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_24_model_states.pt. -[default3]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_23_model_states.pt... -[default7]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_23_model_states.pt. -[default2]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_38_model_states.pt... -[default2]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_38_model_states.pt. -[default4]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_39_model_states.pt... -[default4]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_39_model_states.pt. -[default2]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_44_model_states.pt... -[default2]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_44_model_states.pt. -[default2]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_44_model_states.pt... -[default4]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_25_model_states.pt... -[default4]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_25_model_states.pt. -[default3]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_44_model_states.pt. -[default4]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_38_model_states.pt... -[default1]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_44_model_states.pt... -[default1]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_44_model_states.pt. -[default1]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_38_model_states.pt. -[default1]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_38_model_states.pt... -[default1]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_38_model_states.pt. -[default6]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_23_model_states.pt... -[default6]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_23_model_states.pt. -[default7]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_45_model_states.pt... -[default4]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_45_model_states.pt. -[default7]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_09_model_states.pt... -[default4]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_09_model_states.pt. -[default4]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_09_model_states.pt... -[default7]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_09_model_states.pt... -[default7]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_09_model_states.pt. -[default5]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_09_model_states.pt. -[default5]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_70_model_states.pt... -[default0]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_70_model_states.pt. -[default1]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_70_model_states.pt... -[default1]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_70_model_states.pt. -[default0]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_70_model_states.pt... -[default2]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_70_model_states.pt. -[default2]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_08_model_states.pt... -[default3]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_08_model_states.pt. -[default3]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_55_model_states.pt... -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_55_model_states.pt. -[default6]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_55_model_states.pt... -[default7]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_55_model_states.pt. -[default7]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_35_model_states.pt... -[default4]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_35_model_states.pt. -[default4]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_71_model_states.pt... -[default4]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_71_model_states.pt. -[default3]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_54_model_states.pt... -[default3]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_54_model_states.pt. -[default3]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_45_model_states.pt... -[default5]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_45_model_states.pt. -[default5]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_45_model_states.pt... -[default7]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_45_model_states.pt. -[default7]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_34_model_states.pt... -[default3]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_38_model_states.pt... -[default3]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_38_model_states.pt. -[default6]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_45_model_states.pt... -[default6]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_45_model_states.pt. -[default6]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_34_model_states.pt. -[default2]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_34_model_states.pt... -[default4]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_55_model_states.pt... -[default4]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_55_model_states.pt. -[default4]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_34_model_states.pt... -[default5]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_35_model_states.pt... -[default0]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_34_model_states.pt. -[default0]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_34_model_states.pt... -[default3]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_34_model_states.pt. -[default1]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_34_model_states.pt. -[default1]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_35_model_states.pt. -[default5]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_35_model_states.pt... -[default7]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_35_model_states.pt. -[default7]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_59_model_states.pt... -[default5]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_59_model_states.pt. -[default5]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_59_model_states.pt... -[default6]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_59_model_states.pt. -[default5]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_58_model_states.pt... -[default3]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_58_model_states.pt. -[default3]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_58_model_states.pt... -[default2]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_58_model_states.pt. -[default2]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_24_model_states.pt... -[default1]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_24_model_states.pt. -[default1]:[2022-09-07 22:14:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_58_model_states.pt... -[default0]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_58_model_states.pt. -[default0]:[2022-09-07 22:14:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_26_model_states.pt... -[default0]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_26_model_states.pt. -[default0]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_71_model_states.pt... -[default5]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_71_model_states.pt. -[default4]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_59_model_states.pt... -[default4]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_59_model_states.pt. -[default4]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_58_model_states.pt... -[default1]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_58_model_states.pt. -[default1]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_59_model_states.pt... -[default7]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_59_model_states.pt. -[default7]:[2022-09-07 22:14:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_62_model_states.pt... -[default6]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_39_model_states.pt... -[default7]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_39_model_states.pt. -[default7]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_39_model_states.pt... -[default7]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_39_model_states.pt. -[default7]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_39_model_states.pt... -[default5]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_39_model_states.pt. -[default5]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_70_model_states.pt... -[default3]:[2022-09-07 22:14:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_70_model_states.pt. -[default3]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_63_model_states.pt... -[default0]:[2022-09-07 22:14:36,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_46_model_states.pt... -[default0]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_46_model_states.pt. -[default6]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_21_model_states.pt... -[default0]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_20_model_states.pt... -[default4]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_21_model_states.pt... -[default5]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_21_model_states.pt... -[default3]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_20_model_states.pt... -[default2]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_53_model_states.pt... -[default5]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_53_model_states.pt... -[default4]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_53_model_states.pt... -[default4]:[2022-09-07 22:14:36,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_47_model_states.pt... -[default4]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_47_model_states.pt. -[default4]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_63_model_states.pt... -[default7]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_63_model_states.pt. -[default7]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_21_model_states.pt... -[default0]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_35_model_states.pt... -[default6]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_35_model_states.pt. -[default6]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_61_model_states.pt... -[default4]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_61_model_states.pt. -[default4]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_60_model_states.pt... -[default0]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_60_model_states.pt. -[default0]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_61_model_states.pt... -[default6]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_61_model_states.pt. -[default7]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_61_model_states.pt... -[default6]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_61_model_states.pt. -[default3]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_60_model_states.pt... -[default3]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_60_model_states.pt. -[default3]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_41_model_states.pt... -[default6]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_41_model_states.pt. -[default6]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_53_model_states.pt... -[default7]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_53_model_states.pt. -[default2]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_26_model_states.pt... -[default2]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_26_model_states.pt. -[default2]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_27_model_states.pt... -[default5]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_27_model_states.pt. -[default4]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_27_model_states.pt... -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_27_model_states.pt. -[default4]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_27_model_states.pt... -[default6]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_27_model_states.pt. -[default6]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_26_model_states.pt... -[default3]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_26_model_states.pt. -[default3]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_27_model_states.pt... -[default7]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_27_model_states.pt. -[default7]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_26_model_states.pt... -[default1]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_26_model_states.pt. -[default1]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_23_model_states.pt... -[default4]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_23_model_states.pt. -[default4]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_47_model_states.pt... -[default6]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_47_model_states.pt. -[default6]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_71_model_states.pt... -[default6]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_71_model_states.pt. -[default2]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_46_model_states.pt... -[default2]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_46_model_states.pt. -[default2]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_47_model_states.pt... -[default5]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_47_model_states.pt. -[default1]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_46_model_states.pt... -[default1]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_46_model_states.pt. -[default1]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_46_model_states.pt... -[default3]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_46_model_states.pt. -[default3]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_47_model_states.pt... -[default7]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_47_model_states.pt. -[default7]:[2022-09-07 22:14:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_71_model_states.pt... -[default7]:[2022-09-07 22:14:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_71_model_states.pt. -[default7]:[2022-09-07 22:14:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_13_model_states.pt... -[default7]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_13_model_states.pt. -[default0]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_22_model_states.pt... -[default0]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_22_model_states.pt. -[default0]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_01_model_states.pt... -[default5]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_01_model_states.pt. -[default7]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_01_model_states.pt... -[default7]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_01_model_states.pt. -[default7]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_01_model_states.pt... -[default4]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_01_model_states.pt. -[default4]:[2022-09-07 22:14:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_01_model_states.pt... -[default6]:[2022-09-07 22:14:36,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_01_model_states.pt. -[default6]:[2022-09-07 22:14:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_10_model_states.pt... -[default1]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_10_model_states.pt. -[default1]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_10_model_states.pt... -[default2]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_10_model_states.pt... -[default0]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_10_model_states.pt. -[default0]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_10_model_states.pt. -[default2]:[2022-09-07 22:14:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_10_model_states.pt... -[default3]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_10_model_states.pt. -[default3]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_42_model_states.pt... -[default2]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_42_model_states.pt... -[default4]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_43_model_states.pt... -[default4]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_43_model_states.pt. -[default4]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_42_model_states.pt. -[default1]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_42_model_states.pt. -[default0]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_42_model_states.pt... -[default0]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_42_model_states.pt. -[default0]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_42_model_states.pt... -[default5]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_43_model_states.pt... -[default5]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_43_model_states.pt. -[default5]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_42_model_states.pt. -[default2]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_66_model_states.pt... -[default0]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_66_model_states.pt. -[default0]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_67_model_states.pt... -[default6]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_67_model_states.pt. -[default6]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_11_model_states.pt. -[default7]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_11_model_states.pt. -[default5]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_11_model_states.pt. -[default4]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_11_model_states.pt. -[default4]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_30_model_states.pt... -[default0]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_30_model_states.pt. -[default0]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_30_model_states.pt... -[default1]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_30_model_states.pt. -[default1]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_66_model_states.pt... -[default1]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_66_model_states.pt. -[default2]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_66_model_states.pt... -[default2]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_66_model_states.pt. -[default2]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_67_model_states.pt... -[default5]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_66_model_states.pt... -[default3]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_66_model_states.pt. -[default3]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_67_model_states.pt. -[default4]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_67_model_states.pt... -[default7]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_67_model_states.pt... -[default7]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_67_model_states.pt. -[default7]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_67_model_states.pt. -[default5]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_30_model_states.pt... -[default2]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_30_model_states.pt. -[default2]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_06_model_states.pt... -[default1]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_06_model_states.pt. -[default1]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_56_model_states.pt... -[default6]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_57_model_states.pt... -[default6]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_57_model_states.pt. -[default2]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_56_model_states.pt... -[default1]:[2022-09-07 22:14:36,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_56_model_states.pt. -[default0]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_56_model_states.pt... -[default6]:[2022-09-07 22:14:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_56_model_states.pt. -[default4]:[2022-09-07 22:14:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_56_model_states.pt. -[default2]:[2022-09-07 22:14:36,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_57_model_states.pt... -[default4]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_57_model_states.pt. -[default4]:[2022-09-07 22:14:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_56_model_states.pt... -[default3]:[2022-09-07 22:14:36,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_56_model_states.pt. -[default3]:[2022-09-07 22:14:36,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_06_model_states.pt. -[default0]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_32_model_states.pt... -[default2]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_32_model_states.pt. -[default2]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_30_model_states.pt... -[default3]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_30_model_states.pt. -[default1]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_32_model_states.pt... -[default0]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_32_model_states.pt... -[default3]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_32_model_states.pt. -[default3]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_32_model_states.pt... -[default1]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_32_model_states.pt. -[default0]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_32_model_states.pt. -[default0]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_33_model_states.pt... -[default4]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_33_model_states.pt. -[default4]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_16_model_states.pt... -[default0]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_16_model_states.pt. -[default0]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_44_model_states.pt. -[default0]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_31_model_states.pt... -[default4]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_31_model_states.pt. -[default4]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_31_model_states.pt... -[default6]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_31_model_states.pt. -[default6]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_31_model_states.pt... -[default7]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_31_model_states.pt. -[default7]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_31_model_states.pt... -[default5]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_31_model_states.pt. -[default5]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_16_model_states.pt... -[default3]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_16_model_states.pt. -[default3]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_17_model_states.pt... -[default1]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_17_model_states.pt. -[default1]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_16_model_states.pt... -[default1]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_16_model_states.pt. -[default1]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_16_model_states.pt... -[default2]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_16_model_states.pt. -[default2]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_33_model_states.pt... -[default7]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_33_model_states.pt. -[default7]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_64_model_states.pt... -[default0]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_64_model_states.pt. -[default0]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_64_model_states.pt... -[default5]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_65_model_states.pt... -[default5]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_65_model_states.pt. -[default3]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_64_model_states.pt. -[default2]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_22_model_states.pt. -[default3]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_22_model_states.pt. -[default3]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_64_model_states.pt... -[default2]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_64_model_states.pt. -[default2]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_22_model_states.pt. -[default1]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_64_model_states.pt... -[default1]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_64_model_states.pt. -[default1]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_65_model_states.pt... -[default5]:[2022-09-07 22:14:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_65_model_states.pt. -[default4]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_65_model_states.pt... -[default7]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_65_model_states.pt. -[default6]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_65_model_states.pt... -[default6]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_65_model_states.pt. -[default4]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_23_model_states.pt. -[default5]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_17_model_states.pt... -[default6]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_17_model_states.pt. -[default6]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_33_model_states.pt... -[default5]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_17_model_states.pt... -[default5]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_17_model_states.pt. -[default5]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_33_model_states.pt... -[default5]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_33_model_states.pt. -[default5]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_33_model_states.pt. -[default6]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_17_model_states.pt... -[default7]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_17_model_states.pt. -[default7]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_20_model_states.pt... -[default2]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_20_model_states.pt. -[default2]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_62_model_states.pt... -[default1]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_62_model_states.pt. -[default1]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_63_model_states.pt... -[default4]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_63_model_states.pt. -[default4]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_61_model_states.pt... -[default5]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_61_model_states.pt. -[default5]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_60_model_states.pt... -[default1]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_60_model_states.pt. -[default1]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default4]:[2022-09-07 22:14:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default2]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_12_model_states.pt... -[default4]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default4]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_13_model_states.pt... -[default4]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_13_model_states.pt. -[default4]:[2022-09-07 22:14:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default1]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_12_model_states.pt... -[default2]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_12_model_states.pt. -[default2]:[2022-09-07 22:14:36,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_12_model_states.pt. -[default1]:[2022-09-07 22:14:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default3]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_12_model_states.pt... -[default3]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_12_model_states.pt. -[default3]:[2022-09-07 22:14:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_20_model_states.pt... -[default0]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_20_model_states.pt. -[default0]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_62_model_states.pt... -[default2]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_62_model_states.pt. -[default5]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_63_model_states.pt... -[default5]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_63_model_states.pt. -[default3]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_62_model_states.pt. -[default3]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_52_model_states.pt... -[default0]:[2022-09-07 22:14:36,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_52_model_states.pt. -[default0]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_21_model_states.pt. -[default1]:[2022-09-07 22:14:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default1]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_52_model_states.pt... -[default1]:[2022-09-07 22:14:36,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_52_model_states.pt. -[default1]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default3]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_52_model_states.pt... -[default3]:[2022-09-07 22:14:36,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_52_model_states.pt. -[default3]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_20_model_states.pt. -[default3]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_21_model_states.pt. -[default5]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_21_model_states.pt. -[default4]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_20_model_states.pt. -[default1]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_53_model_states.pt. -[default6]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_52_model_states.pt... -[default2]:[2022-09-07 22:14:36,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_52_model_states.pt. -[default2]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_53_model_states.pt. -[default4]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_53_model_states.pt. -[default5]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_21_model_states.pt. -[default7]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_57_model_states.pt... -[default5]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_57_model_states.pt. -[default5]:[2022-09-07 22:14:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default7]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default7]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_57_model_states.pt... -[default7]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_57_model_states.pt. -[default7]:[2022-09-07 22:14:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default0]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default0]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_12_model_states.pt... -[default0]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_12_model_states.pt. -[default0]:[2022-09-07 22:14:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default5]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default5]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_13_model_states.pt... -[default5]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_13_model_states.pt. -[default5]:[2022-09-07 22:14:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt... -[default6]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default6]:[2022-09-07 22:14:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_13_model_states.pt... -[default6]:[2022-09-07 22:14:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_13_model_states.pt. -[default6]:[2022-09-07 22:14:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_00_model_states.pt. -[default2]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_60_model_states.pt... -[default2]:[2022-09-07 22:14:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_60_model_states.pt. -[default2]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_63_model_states.pt. -[default2]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_62_model_states.pt... -[default6]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:36,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/mp_rank_62_model_states.pt. -[default0]:[2022-09-07 22:14:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:38,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:39,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:38,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:38,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:38,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:38,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:38,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:38,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:38,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:39,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:39,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:39,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:39,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:38,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:38,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:38,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:39,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:38,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:38,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:38,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:38,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:39,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:39,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:39,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:39,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:39,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:39,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:39,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:39,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:39,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:39,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:39,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:39,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:39,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:39,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:39,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:39,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:39,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:39,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:39,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:39,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:39,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:39,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:39,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:39,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:39,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_72-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:39,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:39,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:39,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:39,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:39,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:39,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:39,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:39,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:39,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:39,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:39,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:39,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:39,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:39,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:39,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:39,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:39,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt... -[default6]:[2022-09-07 22:14:39,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:39,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt... -[default7]:[2022-09-07 22:14:39,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt... -[default4]:[2022-09-07 22:14:39,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:39,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:39,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:39,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt... -[default2]:[2022-09-07 22:14:39,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default0]:[2022-09-07 22:14:39,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:39,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt... -[default1]:[2022-09-07 22:14:39,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:39,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default3]:[2022-09-07 22:14:39,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:39,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:39,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_74-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default2]:[2022-09-07 22:14:39,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_22-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_70-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default2]:[2022-09-07 22:14:39,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_54-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:39,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_68-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_16-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:39,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:39,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:39,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_05-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:39,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:39,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_28-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:39,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:39,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:39,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_48-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:39,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_58-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_69-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default4]:[2022-09-07 22:14:40,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_04-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_24-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_25-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_17-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_34-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_62-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default5]:[2022-09-07 22:14:40,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_49-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_55-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_13-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default5]:[2022-09-07 22:14:40,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_35-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_23-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_71-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_21-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_36-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_64-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_29-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default3]:[2022-09-07 22:14:40,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_44-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_06-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default6]:[2022-09-07 22:14:40,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_27-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default7]:[2022-09-07 22:14:40,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default1]:[2022-09-07 22:14:40,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default0]:[2022-09-07 22:14:40,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_12-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default4]:[2022-09-07 22:14:40,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_39-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_46-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_20-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_47-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_57-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default7]:[2022-09-07 22:14:40,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_37-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:40,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_41-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_56-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:40,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_63-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:40,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default7]:[2022-09-07 22:14:40,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default3]:[2022-09-07 22:14:40,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default0]:[2022-09-07 22:14:40,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default0]:[2022-09-07 22:14:40,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default0]:[2022-09-07 22:14:40,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default2]:[2022-09-07 22:14:40,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default0]:[2022-09-07 22:14:40,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default2]:[2022-09-07 22:14:40,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default6]:[2022-09-07 22:14:40,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default2]:[2022-09-07 22:14:40,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:40,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_65-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:40,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default0]:[2022-09-07 22:14:40,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default1]:[2022-09-07 22:14:40,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default3]:[2022-09-07 22:14:40,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default1]:[2022-09-07 22:14:40,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default3]:[2022-09-07 22:14:40,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default1]:[2022-09-07 22:14:40,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default0]:[2022-09-07 22:14:41,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default3]:[2022-09-07 22:14:40,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default1]:[2022-09-07 22:14:40,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_38-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:40,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default3]:[2022-09-07 22:14:41,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default2]:[2022-09-07 22:14:40,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default6]:[2022-09-07 22:14:41,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:40,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default3]:[2022-09-07 22:14:41,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_26-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_60-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:41,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:40,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default5]:[2022-09-07 22:14:41,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default6]:[2022-09-07 22:14:41,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default0]:[2022-09-07 22:14:41,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default1]:[2022-09-07 22:14:41,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default3]:[2022-09-07 22:14:41,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default2]:[2022-09-07 22:14:41,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default6]:[2022-09-07 22:14:41,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:41,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default2]:[2022-09-07 22:14:41,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default0]:[2022-09-07 22:14:41,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default1]:[2022-09-07 22:14:41,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default6]:[2022-09-07 22:14:41,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default7]:[2022-09-07 22:14:41,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default4]:[2022-09-07 22:14:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default1]:[2022-09-07 22:14:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_07-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:41,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:41,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_19-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_45-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:41,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_10-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_40-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_61-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_11-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default7]:[2022-09-07 22:14:41,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default2]:[2022-09-07 22:14:41,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default0]:[2022-09-07 22:14:41,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default4]:[2022-09-07 22:14:41,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default6]:[2022-09-07 22:14:41,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:41,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:41,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:41,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_42-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default4]:[2022-09-07 22:14:41,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default1]:[2022-09-07 22:14:41,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default2]:[2022-09-07 22:14:41,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default7]:[2022-09-07 22:14:41,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default1]:[2022-09-07 22:14:41,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default1]:[2022-09-07 22:14:41,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default0]:[2022-09-07 22:14:41,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default4]:[2022-09-07 22:14:41,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default7]:[2022-09-07 22:14:41,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default5]:[2022-09-07 22:14:41,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:41,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default7]:[2022-09-07 22:14:41,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default5]:[2022-09-07 22:14:41,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default6]:[2022-09-07 22:14:41,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default2]:[2022-09-07 22:14:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_33-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_66-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default6]:[2022-09-07 22:14:41,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default4]:[2022-09-07 22:14:41,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default0]:[2022-09-07 22:14:41,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:41,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default5]:[2022-09-07 22:14:41,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default1]:[2022-09-07 22:14:41,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default2]:[2022-09-07 22:14:41,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt... -[default3]:[2022-09-07 22:14:41,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:41,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default3]:[2022-09-07 22:14:41,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default1]:[2022-09-07 22:14:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_18-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default0]:[2022-09-07 22:14:41,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default0]:[2022-09-07 22:14:41,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt... -[default5]:[2022-09-07 22:14:41,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default2]:[2022-09-07 22:14:41,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default3]:[2022-09-07 22:14:41,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default1]:[2022-09-07 22:14:41,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt... -[default0]:[2022-09-07 22:14:41,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:41,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_30-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default3]:[2022-09-07 22:14:41,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_52-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:41,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default3]:[2022-09-07 22:14:41,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default2]:[2022-09-07 22:14:41,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default6]:[2022-09-07 22:14:41,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:41,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:41,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_67-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default2]:[2022-09-07 22:14:41,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default1]:[2022-09-07 22:14:41,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default3]:[2022-09-07 22:14:41,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default7]:[2022-09-07 22:14:41,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default6]:[2022-09-07 22:14:41,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default7]:[2022-09-07 22:14:41,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:41,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default1]:[2022-09-07 22:14:41,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:41,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_53-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:41,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default2]:[2022-09-07 22:14:41,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default0]:[2022-09-07 22:14:41,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default4]:[2022-09-07 22:14:41,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:41,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_09-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:41,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default5]:[2022-09-07 22:14:41,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default0]:[2022-09-07 22:14:41,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:41,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default2]:[2022-09-07 22:14:41,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default0]:[2022-09-07 22:14:41,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default3]:[2022-09-07 22:14:41,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default4]:[2022-09-07 22:14:41,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default6]:[2022-09-07 22:14:41,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default7]:[2022-09-07 22:14:41,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default1]:[2022-09-07 22:14:41,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default3]:[2022-09-07 22:14:41,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default5]:[2022-09-07 22:14:41,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default2]:[2022-09-07 22:14:41,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default7]:[2022-09-07 22:14:41,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:41,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default0]:[2022-09-07 22:14:41,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default1]:[2022-09-07 22:14:41,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default2]:[2022-09-07 22:14:41,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default6]:[2022-09-07 22:14:41,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default7]:[2022-09-07 22:14:41,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default0]:[2022-09-07 22:14:41,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default5]:[2022-09-07 22:14:41,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default6]:[2022-09-07 22:14:41,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_43-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:41,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default0]:[2022-09-07 22:14:41,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default5]:[2022-09-07 22:14:41,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default6]:[2022-09-07 22:14:41,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default4]:[2022-09-07 22:14:41,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default3]:[2022-09-07 22:14:41,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default0]:[2022-09-07 22:14:41,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default3]:[2022-09-07 22:14:41,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_32-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:41,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default3]:[2022-09-07 22:14:41,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default3]:[2022-09-07 22:14:41,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default6]:[2022-09-07 22:14:41,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default3]:[2022-09-07 22:14:41,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default6]:[2022-09-07 22:14:41,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default7]:[2022-09-07 22:14:41,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default4]:[2022-09-07 22:14:41,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default7]:[2022-09-07 22:14:42,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default6]:[2022-09-07 22:14:42,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default0]:[2022-09-07 22:14:42,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default3]:[2022-09-07 22:14:41,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_08-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:41,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default1]:[2022-09-07 22:14:42,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default1]:[2022-09-07 22:14:41,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default5]:[2022-09-07 22:14:42,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default7]:[2022-09-07 22:14:41,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default5]:[2022-09-07 22:14:42,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default4]:[2022-09-07 22:14:42,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default4]:[2022-09-07 22:14:42,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default5]:[2022-09-07 22:14:42,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default5]:[2022-09-07 22:14:42,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default3]:[2022-09-07 22:14:42,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default4]:[2022-09-07 22:14:42,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default3]:[2022-09-07 22:14:42,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default3]:[2022-09-07 22:14:42,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default1]:[2022-09-07 22:14:42,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default2]:[2022-09-07 22:14:42,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default4]:[2022-09-07 22:14:42,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:42,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:42,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:42,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_31-model_00-model_states.pt. -[default0]:[2022-09-07 22:14:42,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default1]:[2022-09-07 22:14:42,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default1]:[2022-09-07 22:14:42,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default0]:[2022-09-07 22:14:42,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default0]:[2022-09-07 22:14:42,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default0]:[2022-09-07 22:14:42,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:42,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:42,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default2]:[2022-09-07 22:14:42,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default2]:[2022-09-07 22:14:42,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default0]:[2022-09-07 22:14:42,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default3]:[2022-09-07 22:14:42,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default3]:[2022-09-07 22:14:42,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default5]:[2022-09-07 22:14:42,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default3]:[2022-09-07 22:14:42,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:42,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:42,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default6]:[2022-09-07 22:14:42,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default4]:[2022-09-07 22:14:42,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default0]:[2022-09-07 22:14:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_14-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:42,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default1]:[2022-09-07 22:14:42,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:42,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_50-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:42,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default0]:[2022-09-07 22:14:42,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default6]:[2022-09-07 22:14:42,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default2]:[2022-09-07 22:14:42,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default4]:[2022-09-07 22:14:42,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default0]:[2022-09-07 22:14:42,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default5]:[2022-09-07 22:14:42,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default6]:[2022-09-07 22:14:42,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default7]:[2022-09-07 22:14:42,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default2]:[2022-09-07 22:14:42,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default6]:[2022-09-07 22:14:42,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default4]:[2022-09-07 22:14:42,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default5]:[2022-09-07 22:14:42,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:42,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:42,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default3]:[2022-09-07 22:14:42,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default6]:[2022-09-07 22:14:42,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default1]:[2022-09-07 22:14:42,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default4]:[2022-09-07 22:14:42,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default5]:[2022-09-07 22:14:42,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default7]:[2022-09-07 22:14:42,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default7]:[2022-09-07 22:14:42,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_59-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:42,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default1]:[2022-09-07 22:14:42,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default0]:[2022-09-07 22:14:42,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default7]:[2022-09-07 22:14:42,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default7]:[2022-09-07 22:14:42,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default0]:[2022-09-07 22:14:42,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default6]:[2022-09-07 22:14:42,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default4]:[2022-09-07 22:14:42,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default5]:[2022-09-07 22:14:42,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default6]:[2022-09-07 22:14:42,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default7]:[2022-09-07 22:14:42,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default7]:[2022-09-07 22:14:42,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default7]:[2022-09-07 22:14:42,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default6]:[2022-09-07 22:14:42,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default7]:[2022-09-07 22:14:42,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default7]:[2022-09-07 22:14:42,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default4]:[2022-09-07 22:14:42,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default5]:[2022-09-07 22:14:42,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default6]:[2022-09-07 22:14:42,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default2]:[2022-09-07 22:14:42,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default4]:[2022-09-07 22:14:42,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:42,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default4]:[2022-09-07 22:14:42,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default5]:[2022-09-07 22:14:42,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default1]:[2022-09-07 22:14:42,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default0]:[2022-09-07 22:14:42,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default7]:[2022-09-07 22:14:42,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default4]:[2022-09-07 22:14:42,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default5]:[2022-09-07 22:14:42,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default4]:[2022-09-07 22:14:42,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default6]:[2022-09-07 22:14:42,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default7]:[2022-09-07 22:14:42,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default4]:[2022-09-07 22:14:42,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default5]:[2022-09-07 22:14:42,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default1]:[2022-09-07 22:14:42,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default5]:[2022-09-07 22:14:42,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default4]:[2022-09-07 22:14:42,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default2]:[2022-09-07 22:14:42,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default6]:[2022-09-07 22:14:42,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default7]:[2022-09-07 22:14:42,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default7]:[2022-09-07 22:14:42,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default0]:[2022-09-07 22:14:42,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default6]:[2022-09-07 22:14:42,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_51-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:42,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default2]:[2022-09-07 22:14:42,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default3]:[2022-09-07 22:14:42,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default0]:[2022-09-07 22:14:42,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default5]:[2022-09-07 22:14:42,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default4]:[2022-09-07 22:14:42,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default4]:[2022-09-07 22:14:42,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt. -[default5]:[2022-09-07 22:14:42,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:42,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:42,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default3]:[2022-09-07 22:14:42,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default7]:[2022-09-07 22:14:43,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_15-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:43,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default2]:[2022-09-07 22:14:43,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default1]:[2022-09-07 22:14:43,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default6]:[2022-09-07 22:14:43,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default5]:[2022-09-07 22:14:43,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default2]:[2022-09-07 22:14:43,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default4]:[2022-09-07 22:14:43,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default3]:[2022-09-07 22:14:43,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default0]:[2022-09-07 22:14:43,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default3]:[2022-09-07 22:14:43,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default5]:[2022-09-07 22:14:43,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default7]:[2022-09-07 22:14:43,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default7]:[2022-09-07 22:14:43,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default6]:[2022-09-07 22:14:43,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default5]:[2022-09-07 22:14:43,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default4]:[2022-09-07 22:14:43,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default4]:[2022-09-07 22:14:43,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default6]:[2022-09-07 22:14:43,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default0]:[2022-09-07 22:14:43,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default2]:[2022-09-07 22:14:43,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default6]:[2022-09-07 22:14:43,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default1]:[2022-09-07 22:14:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default5]:[2022-09-07 22:14:43,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:43,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt. -[default4]:[2022-09-07 22:14:43,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt. -[default6]:[2022-09-07 22:14:43,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_03-model_00-model_states.pt. -[default7]:[2022-09-07 22:14:43,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default7]:[2022-09-07 22:14:43,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default5]:[2022-09-07 22:14:43,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default4]:[2022-09-07 22:14:43,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default4]:[2022-09-07 22:14:43,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default5]:[2022-09-07 22:14:43,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default6]:[2022-09-07 22:14:43,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default1]:[2022-09-07 22:14:43,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default5]:[2022-09-07 22:14:43,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default4]:[2022-09-07 22:14:43,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default5]:[2022-09-07 22:14:44,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default4]:[2022-09-07 22:14:44,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default7]:[2022-09-07 22:14:44,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default6]:[2022-09-07 22:14:44,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default7]:[2022-09-07 22:14:44,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default6]:[2022-09-07 22:14:44,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default4]:[2022-09-07 22:14:44,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default5]:[2022-09-07 22:14:44,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default7]:[2022-09-07 22:14:44,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default6]:[2022-09-07 22:14:44,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default5]:[2022-09-07 22:14:44,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default4]:[2022-09-07 22:14:44,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default2]:[2022-09-07 22:14:45,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-07 22:14:45,139] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 210 -[default1]:[2022-09-07 22:14:45,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-07 22:14:45,732] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 225 -[default1]:[2022-09-07 22:14:46,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-07 22:14:46,389] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 89 -[default2]:[2022-09-07 22:14:46,427] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 210 -[default2]:[2022-09-07 22:14:46,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-07 22:14:46,474] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 282 -[default1]:[2022-09-07 22:14:46,791] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 225 -[default3]:[2022-09-07 22:14:47,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-07 22:14:47,313] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 227 -[default0]:[2022-09-07 22:14:47,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-07 22:14:47,443] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 280 -[default1]:[2022-09-07 22:14:47,442] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 89 -[default2]:[2022-09-07 22:14:47,589] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 282 -[default1]:[2022-09-07 22:14:47,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-07 22:14:47,857] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 281 -[default4]:[2022-09-07 22:14:48,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-07 22:14:48,054] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 228 -[default3]:[2022-09-07 22:14:48,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-07 22:14:48,171] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 283 -[default2]:[2022-09-07 22:14:48,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-07 22:14:48,270] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 90 -[default0]:[2022-09-07 22:14:48,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-07 22:14:48,428] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 88 -[default0]:[2022-09-07 22:14:48,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-07 22:14:48,551] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 224 -[default3]:[2022-09-07 22:14:48,532] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 227 -[default0]:[2022-09-07 22:14:48,615] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 280 -[default3]:[2022-09-07 22:14:48,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-07 22:14:48,870] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 187 -[default1]:[2022-09-07 22:14:48,956] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 281 -[default0]:[2022-09-07 22:14:49,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-07 22:14:49,113] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 8 -[default4]:[2022-09-07 22:14:49,135] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 228 -[default0]:[2022-09-07 22:14:49,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt. -[default1]:[2022-09-07 22:14:49,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt. -[default2]:[2022-09-07 22:14:49,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:49,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/layer_01-model_00-model_states.pt. -[default3]:[2022-09-07 22:14:49,260] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 283 -[default3]:[2022-09-07 22:14:49,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-07 22:14:49,318] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 267 -[default2]:[2022-09-07 22:14:49,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-07 22:14:49,284] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 10 -[default0]:[2022-09-07 22:14:49,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-07 22:14:49,368] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 80 -[default1]:[2022-09-07 22:14:49,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-07 22:14:49,372] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 9 -[default2]:[2022-09-07 22:14:49,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-07 22:14:49,527] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 226 -[default0]:[2022-09-07 22:14:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-07 22:14:49,502] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 208 -[default2]:[2022-09-07 22:14:49,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-07 22:14:49,555] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 186 -[default2]:[2022-09-07 22:14:49,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-07 22:14:49,573] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 266 -[default0]:[2022-09-07 22:14:49,639] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 224 -[default3]:[2022-09-07 22:14:49,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-07 22:14:49,701] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 275 -[default2]:[2022-09-07 22:14:49,724] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 90 -[default0]:[2022-09-07 22:14:49,687] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 88 -[default6]:[2022-09-07 22:14:50,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-07 22:14:50,087] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 230 -[default3]:[2022-09-07 22:14:50,086] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 187 -[default2]:[2022-09-07 22:14:50,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-07 22:14:50,303] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 106 -[default0]:[2022-09-07 22:14:50,408] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 8 -[default0]:[2022-09-07 22:14:50,418] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 80 -[default2]:[2022-09-07 22:14:50,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-07 22:14:50,468] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 274 -[default3]:[2022-09-07 22:14:50,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-07 22:14:50,481] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 83 -[default0]:[2022-09-07 22:14:50,569] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 208 -[default4]:[2022-09-07 22:14:50,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-07 22:14:50,481] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 188 -[default1]:[2022-09-07 22:14:50,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-07 22:14:50,559] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 241 -[default1]:[2022-09-07 22:14:50,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-07 22:14:50,510] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 105 -[default2]:[2022-09-07 22:14:50,615] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 226 -[default2]:[2022-09-07 22:14:50,614] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 186 -[default3]:[2022-09-07 22:14:50,731] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 267 -[default1]:[2022-09-07 22:14:50,751] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 9 -[default3]:[2022-09-07 22:14:50,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-07 22:14:50,752] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 259 -[default3]:[2022-09-07 22:14:50,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-07 22:14:50,745] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 107 -[default1]:[2022-09-07 22:14:50,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-07 22:14:50,767] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 273 -[default0]:[2022-09-07 22:14:50,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-07 22:14:50,872] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 184 -[default0]:[2022-09-07 22:14:50,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-07 22:14:50,917] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 272 -[default0]:[2022-09-07 22:14:50,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-07 22:14:50,923] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 264 -[default2]:[2022-09-07 22:14:50,923] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 266 -[default3]:[2022-09-07 22:14:50,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-07 22:14:50,895] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 131 -[default2]:[2022-09-07 22:14:50,922] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 10 -[default6]:[2022-09-07 22:14:50,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-07 22:14:50,869] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 94 -[default1]:[2022-09-07 22:14:50,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-07 22:14:50,929] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 209 -[default4]:[2022-09-07 22:14:50,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-07 22:14:50,957] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 92 -[default1]:[2022-09-07 22:14:50,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-07 22:14:50,993] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 129 -[default2]:[2022-09-07 22:14:51,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-07 22:14:51,022] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 130 -[default5]:[2022-09-07 22:14:51,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-07 22:14:51,062] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 13 -[default0]:[2022-09-07 22:14:51,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-07 22:14:51,062] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 256 -[default4]:[2022-09-07 22:14:50,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-07 22:14:50,982] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 212 -[default7]:[2022-09-07 22:14:50,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-07 22:14:50,985] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 191 -[default1]:[2022-09-07 22:14:51,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-07 22:14:51,066] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 185 -[default3]:[2022-09-07 22:14:51,085] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 275 -[default3]:[2022-09-07 22:14:51,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-07 22:14:51,083] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 11 -[default0]:[2022-09-07 22:14:51,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-07 22:14:51,134] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 104 -[default3]:[2022-09-07 22:14:51,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-07 22:14:51,094] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 243 -[default3]:[2022-09-07 22:14:51,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-07 22:14:51,217] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 59 -[default6]:[2022-09-07 22:14:51,203] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 230 -[default6]:[2022-09-07 22:14:51,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-07 22:14:51,165] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 14 -[default7]:[2022-09-07 22:14:51,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-07 22:14:51,182] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 271 -[default5]:[2022-09-07 22:14:51,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-07 22:14:51,269] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 85 -[default6]:[2022-09-07 22:14:51,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default7]:[2022-09-07 22:14:51,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-07 22:14:51,228] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 231 -[default3]:[2022-09-07 22:14:51,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-07 22:14:51,188] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 211 -[default2]:[2022-09-07 22:14:51,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default2]:[2022-09-07 22:14:51,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-07 22:14:51,315] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 82 -[default0]:[2022-09-07 22:14:51,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-07 22:14:51,366] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 240 -[default7]:[2022-09-07 22:14:51,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default5]:[2022-09-07 22:14:51,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-07 22:14:51,333] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 189 -[default3]:[2022-09-07 22:14:51,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default2]:[2022-09-07 22:14:51,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-07 22:14:51,367] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 258 -[default5]:[2022-09-07 22:14:51,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default6]:[2022-09-07 22:14:51,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-07 22:14:51,411] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 134 -[default1]:[2022-09-07 22:14:51,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default0]:[2022-09-07 22:14:51,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-07 22:14:51,512] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 56 -[default4]:[2022-09-07 22:14:51,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default0]: > using checkpoint value 2e-05 for learning rate -[default0]: > using checkpoint value 0.0 for minimum learning rate -[default0]: > using checkpoint value 0 for warmup iterations -[default0]: > using checkpoint value 6348800 for total number of iterations -[default0]: > using checkpoint value constant for decay style -[default0]:[2022-09-07 22:14:51,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default4]:[2022-09-07 22:14:51,524] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 188 -[default2]:[2022-09-07 22:14:51,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-07 22:14:51,653] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 58 -[default0]:[2022-09-07 22:14:51,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-07 22:14:51,657] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 128 -[default3]:[2022-09-07 22:14:51,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-07 22:14:51,574] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 91 -[default7]:[2022-09-07 22:14:51,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-07 22:14:51,666] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 87 -[default6]:[2022-09-07 22:14:51,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-07 22:14:51,588] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 214 -[default6]:[2022-09-07 22:14:51,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-07 22:14:51,643] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 190 -[default1]:[2022-09-07 22:14:51,729] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 241 -[default1]:[2022-09-07 22:14:51,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-07 22:14:51,807] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 57 -[default5]:[2022-09-07 22:14:51,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-07 22:14:51,822] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 157 -[default2]:[2022-09-07 22:14:51,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-07 22:14:51,862] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 242 -[default6]:[2022-09-07 22:14:51,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-07 22:14:51,818] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 86 -[default6]:[2022-09-07 22:14:51,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-07 22:14:51,946] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 46 -[default5]:[2022-09-07 22:14:51,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-07 22:14:51,874] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 221 -[default3]:[2022-09-07 22:14:51,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-07 22:14:51,867] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 75 -[default5]:[2022-09-07 22:14:51,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-07 22:14:51,937] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 133 -[default2]:[2022-09-07 22:14:51,885] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 106 -[default3]:[2022-09-07 22:14:51,898] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 83 -[default0]:[2022-09-07 22:14:52,055] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 264 -[default7]:[2022-09-07 22:14:51,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-07 22:14:51,985] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 15 -[default7]:[2022-09-07 22:14:52,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-07 22:14:52,050] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 263 -[default5]:[2022-09-07 22:14:52,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-07 22:14:52,024] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 93 -[default0]:[2022-09-07 22:14:52,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-07 22:14:52,051] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 136 -[default1]:[2022-09-07 22:14:52,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-07 22:14:52,013] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 137 -[default4]:[2022-09-07 22:14:52,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-07 22:14:52,010] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 84 -[default7]:[2022-09-07 22:14:52,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-07 22:14:52,151] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 279 -[default0]:[2022-09-07 22:14:52,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-07 22:14:52,158] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 192 -[default0]:[2022-09-07 22:14:52,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-07 22:14:52,112] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 16 -[default1]:[2022-09-07 22:14:52,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-07 22:14:52,135] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 257 -[default6]:[2022-09-07 22:14:52,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-07 22:14:52,146] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 262 -[default7]:[2022-09-07 22:14:52,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-07 22:14:52,078] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 95 -[default6]:[2022-09-07 22:14:52,116] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 94 -[default1]:[2022-09-07 22:14:52,148] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 105 -[default1]:[2022-09-07 22:14:52,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-07 22:14:52,158] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 81 -[default1]:[2022-09-07 22:14:52,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-07 22:14:52,244] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 265 -[default7]:[2022-09-07 22:14:52,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-07 22:14:52,259] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 135 -[default4]:[2022-09-07 22:14:52,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-07 22:14:52,276] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 132 -[default3]:[2022-09-07 22:14:52,276] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 259 -[default1]:[2022-09-07 22:14:52,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-07 22:14:52,282] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 217 -[default4]:[2022-09-07 22:14:52,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-07 22:14:52,342] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 220 -[default7]:[2022-09-07 22:14:52,363] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 271 -[default0]:[2022-09-07 22:14:52,320] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 184 -[default7]:[2022-09-07 22:14:52,289] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 231 -[default3]:[2022-09-07 22:14:52,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-07 22:14:52,434] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 43 -[default2]:[2022-09-07 22:14:52,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-07 22:14:52,400] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 42 -[default5]:[2022-09-07 22:14:52,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-07 22:14:52,388] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 269 -[default4]:[2022-09-07 22:14:52,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-07 22:14:52,451] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 60 -[default0]:[2022-09-07 22:14:52,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-07 22:14:52,436] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 72 -[default2]:[2022-09-07 22:14:52,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-07 22:14:52,440] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 250 -[default5]:[2022-09-07 22:14:52,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-07 22:14:52,471] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 213 -[default5]:[2022-09-07 22:14:52,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-07 22:14:52,395] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 109 -[default4]:[2022-09-07 22:14:52,426] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 92 -[default1]:[2022-09-07 22:14:52,511] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 273 -[default2]:[2022-09-07 22:14:52,551] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 274 -[default6]:[2022-09-07 22:14:52,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-07 22:14:52,517] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 270 -[default5]:[2022-09-07 22:14:52,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-07 22:14:52,528] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 61 -[default3]:[2022-09-07 22:14:52,511] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 11 -[default4]:[2022-09-07 22:14:52,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-07 22:14:52,463] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 12 -[default4]:[2022-09-07 22:14:52,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-07 22:14:52,552] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 44 -[default0]:[2022-09-07 22:14:52,568] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 104 -[default1]:[2022-09-07 22:14:52,485] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 209 -[default4]:[2022-09-07 22:14:52,570] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 212 -[default0]:[2022-09-07 22:14:52,634] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 272 -[default3]:[2022-09-07 22:14:52,656] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 59 -[default5]:[2022-09-07 22:14:52,575] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 13 -[default0]:[2022-09-07 22:14:52,577] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 256 -[default6]:[2022-09-07 22:14:52,570] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 14 -[default2]:[2022-09-07 22:14:52,584] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 82 -[default5]:[2022-09-07 22:14:52,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-07 22:14:52,623] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 229 -[default3]:[2022-09-07 22:14:52,627] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 107 -[default1]:[2022-09-07 22:14:52,617] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 185 -[default3]:[2022-09-07 22:14:52,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-07 22:14:52,688] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 19 -[default2]:[2022-09-07 22:14:52,700] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 258 -[default0]:[2022-09-07 22:14:52,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-07 22:14:52,762] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 24 -[default2]:[2022-09-07 22:14:52,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-07 22:14:52,713] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 218 -[default3]:[2022-09-07 22:14:52,732] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 243 -[default6]:[2022-09-07 22:14:52,694] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 214 -[default7]:[2022-09-07 22:14:52,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-07 22:14:52,773] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 215 -[default7]:[2022-09-07 22:14:52,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-07 22:14:52,839] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 47 -[default0]:[2022-09-07 22:14:52,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-07 22:14:52,774] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 160 -[default1]:[2022-09-07 22:14:52,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-07 22:14:52,817] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 17 -[default4]:[2022-09-07 22:14:52,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-07 22:14:52,807] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 156 -[default0]:[2022-09-07 22:14:52,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-07 22:14:52,790] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 216 -[default3]:[2022-09-07 22:14:52,822] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 91 -[default0]:[2022-09-07 22:14:52,781] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 240 -[default7]:[2022-09-07 22:14:52,826] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 191 -[default4]:[2022-09-07 22:14:52,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-07 22:14:52,925] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 268 -[default3]:[2022-09-07 22:14:52,900] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 131 -[default3]:[2022-09-07 22:14:52,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-07 22:14:52,882] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 163 -[default7]:[2022-09-07 22:14:52,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-07 22:14:52,953] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 223 -[default7]:[2022-09-07 22:14:52,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-07 22:14:52,922] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 159 -[default4]:[2022-09-07 22:14:52,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-07 22:14:52,970] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 244 -[default5]:[2022-09-07 22:14:52,927] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 189 -[default6]:[2022-09-07 22:14:52,986] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 190 -[default3]:[2022-09-07 22:14:52,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-07 22:14:52,985] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 171 -[default7]:[2022-09-07 22:14:53,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-07 22:14:53,060] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 63 -[default2]:[2022-09-07 22:14:53,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-07 22:14:53,002] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 170 -[default1]:[2022-09-07 22:14:53,021] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 129 -[default2]:[2022-09-07 22:14:52,968] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 130 -[default2]:[2022-09-07 22:14:53,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-07 22:14:53,019] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 194 -[default3]:[2022-09-07 22:14:53,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-07 22:14:53,061] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 179 -[default5]:[2022-09-07 22:14:53,052] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 221 -[default5]:[2022-09-07 22:14:53,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-07 22:14:53,035] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 45 -[default1]:[2022-09-07 22:14:53,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-07 22:14:53,004] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 153 -[default3]:[2022-09-07 22:14:53,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-07 22:14:53,055] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 51 -[default6]:[2022-09-07 22:14:53,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-07 22:14:53,013] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 110 -[default3]:[2022-09-07 22:14:53,038] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 211 -[default6]:[2022-09-07 22:14:53,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-07 22:14:53,099] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 278 -[default0]:[2022-09-07 22:14:53,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-07 22:14:53,092] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 168 -[default1]:[2022-09-07 22:14:53,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-07 22:14:53,072] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 169 -[default0]:[2022-09-07 22:14:53,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-07 22:14:53,120] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 176 -[default2]:[2022-09-07 22:14:53,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-07 22:14:53,085] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 162 -[default4]:[2022-09-07 22:14:53,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-07 22:14:53,099] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 260 -[default2]:[2022-09-07 22:14:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-07 22:14:53,112] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 234 -[default2]:[2022-09-07 22:14:53,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-07 22:14:53,091] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 74 -[default6]:[2022-09-07 22:14:53,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-07 22:14:53,127] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 222 -[default5]:[2022-09-07 22:14:53,101] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 157 -[default3]:[2022-09-07 22:14:53,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-07 22:14:53,093] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 251 -[default7]:[2022-09-07 22:14:53,242] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 279 -[default0]:[2022-09-07 22:14:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-07 22:14:53,220] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 120 -[default6]:[2022-09-07 22:14:53,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-07 22:14:53,183] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 62 -[default6]:[2022-09-07 22:14:53,221] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 46 -[default7]:[2022-09-07 22:14:53,191] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 15 -[default2]:[2022-09-07 22:14:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-07 22:14:53,220] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 98 -[default0]:[2022-09-07 22:14:53,252] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 16 -[default1]:[2022-09-07 22:14:53,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-07 22:14:53,207] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 73 -[default0]:[2022-09-07 22:14:53,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-07 22:14:53,205] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 248 -[default3]:[2022-09-07 22:14:53,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-07 22:14:53,266] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 219 -[default0]:[2022-09-07 22:14:53,266] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 56 -[default2]:[2022-09-07 22:14:53,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-07 22:14:53,342] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 18 -[default3]:[2022-09-07 22:14:53,288] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 75 -[default3]:[2022-09-07 22:14:53,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-07 22:14:53,327] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 123 -[default6]:[2022-09-07 22:14:53,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-07 22:14:53,347] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 246 -[default6]:[2022-09-07 22:14:53,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-07 22:14:53,369] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 142 -[default1]:[2022-09-07 22:14:53,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-07 22:14:53,385] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 41 -[default0]:[2022-09-07 22:14:53,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-07 22:14:53,437] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 40 -[default5]:[2022-09-07 22:14:53,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-07 22:14:53,371] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 277 -[default0]:[2022-09-07 22:14:53,371] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 192 -[default1]:[2022-09-07 22:14:53,369] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 257 -[default2]:[2022-09-07 22:14:53,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-07 22:14:53,369] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 178 -[default3]:[2022-09-07 22:14:53,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-07 22:14:53,420] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 99 -[default6]:[2022-09-07 22:14:53,440] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 134 -[default5]:[2022-09-07 22:14:53,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-07 22:14:53,454] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 245 -[default5]:[2022-09-07 22:14:53,412] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 93 -[default3]:[2022-09-07 22:14:53,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-07 22:14:53,450] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 139 -[default1]:[2022-09-07 22:14:53,452] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 137 -[default4]:[2022-09-07 22:14:53,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-07 22:14:53,530] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 276 -[default1]:[2022-09-07 22:14:53,544] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 265 -[default4]:[2022-09-07 22:14:53,553] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 12 -[default7]:[2022-09-07 22:14:53,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-07 22:14:53,538] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 167 -[default5]:[2022-09-07 22:14:53,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-07 22:14:53,541] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 101 -[default5]:[2022-09-07 22:14:53,483] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 133 -[default4]:[2022-09-07 22:14:53,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-07 22:14:53,559] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 180 -[default5]:[2022-09-07 22:14:53,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-07 22:14:53,492] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 141 -[default5]:[2022-09-07 22:14:53,523] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 85 -[default7]:[2022-09-07 22:14:53,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-07 22:14:53,507] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 111 -[default1]:[2022-09-07 22:14:53,549] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 81 -[default3]:[2022-09-07 22:14:53,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-07 22:14:53,622] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 195 -[default1]:[2022-09-07 22:14:53,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-07 22:14:53,579] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 161 -[default7]:[2022-09-07 22:14:53,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-07 22:14:53,631] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 79 -[default5]:[2022-09-07 22:14:53,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-07 22:14:53,639] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 261 -[default2]:[2022-09-07 22:14:53,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-07 22:14:53,627] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 138 -[default0]:[2022-09-07 22:14:53,575] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 136 -[default0]:[2022-09-07 22:14:53,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-07 22:14:53,635] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 96 -[default2]:[2022-09-07 22:14:53,673] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 250 -[default5]:[2022-09-07 22:14:53,629] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 213 -[default7]:[2022-09-07 22:14:53,614] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 87 -[default0]:[2022-09-07 22:14:53,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-07 22:14:53,638] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 48 -[default4]:[2022-09-07 22:14:53,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-07 22:14:53,609] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 108 -[default1]:[2022-09-07 22:14:53,682] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 57 -[default1]:[2022-09-07 22:14:53,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-07 22:14:53,716] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 121 -[default2]:[2022-09-07 22:14:53,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-07 22:14:53,729] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 146 -[default7]:[2022-09-07 22:14:53,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-07 22:14:53,746] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 127 -[default0]:[2022-09-07 22:14:53,699] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 72 -[default6]:[2022-09-07 22:14:53,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-07 22:14:53,705] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 30 -[default1]:[2022-09-07 22:14:53,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-07 22:14:53,668] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 177 -[default0]:[2022-09-07 22:14:53,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-07 22:14:53,705] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 152 -[default6]:[2022-09-07 22:14:53,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-07 22:14:53,735] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 38 -[default7]:[2022-09-07 22:14:53,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-07 22:14:53,757] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 103 -[default6]:[2022-09-07 22:14:53,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-07 22:14:53,729] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 182 -[default7]:[2022-09-07 22:14:53,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-07 22:14:53,683] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 183 -[default2]:[2022-09-07 22:14:53,679] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 242 -[default5]:[2022-09-07 22:14:53,714] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 229 -[default7]:[2022-09-07 22:14:53,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-07 22:14:53,683] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 247 -[default5]:[2022-09-07 22:14:53,751] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 109 -[default2]:[2022-09-07 22:14:53,851] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 42 -[default6]:[2022-09-07 22:14:53,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-07 22:14:53,823] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 206 -[default5]:[2022-09-07 22:14:53,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-07 22:14:53,821] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 69 -[default2]:[2022-09-07 22:14:53,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-07 22:14:53,783] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 154 -[default1]:[2022-09-07 22:14:53,848] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 217 -[default0]:[2022-09-07 22:14:53,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-07 22:14:53,868] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 32 -[default7]:[2022-09-07 22:14:53,787] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 263 -[default3]:[2022-09-07 22:14:53,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-07 22:14:53,788] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 235 -[default0]:[2022-09-07 22:14:53,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-07 22:14:53,779] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 232 -[default3]:[2022-09-07 22:14:53,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-07 22:14:53,860] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 155 -[default0]:[2022-09-07 22:14:53,863] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 24 -[default4]:[2022-09-07 22:14:53,797] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 220 -[default6]:[2022-09-07 22:14:53,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-07 22:14:53,819] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 158 -[default4]:[2022-09-07 22:14:53,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-07 22:14:53,867] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 36 -[default1]:[2022-09-07 22:14:53,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-07 22:14:53,804] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 249 -[default1]:[2022-09-07 22:14:53,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-07 22:14:53,784] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 97 -[default6]:[2022-09-07 22:14:53,955] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 270 -[default6]:[2022-09-07 22:14:53,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-07 22:14:53,962] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 126 -[default2]:[2022-09-07 22:14:53,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-07 22:14:53,930] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 66 -[default6]:[2022-09-07 22:14:53,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-07 22:14:53,944] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 102 -[default6]:[2022-09-07 22:14:53,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-07 22:14:53,876] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 78 -[default4]:[2022-09-07 22:14:53,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-07 22:14:53,889] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 76 -[default4]:[2022-09-07 22:14:53,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-07 22:14:53,871] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 140 -[default4]:[2022-09-07 22:14:53,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-07 22:14:53,893] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 124 -[default6]:[2022-09-07 22:14:53,899] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 262 -[default1]:[2022-09-07 22:14:53,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-07 22:14:53,965] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 65 -[default1]:[2022-09-07 22:14:53,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-07 22:14:53,900] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 233 -[default7]:[2022-09-07 22:14:53,883] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 95 -[default5]:[2022-09-07 22:14:53,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-07 22:14:53,973] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 253 -[default7]:[2022-09-07 22:14:53,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-07 22:14:53,895] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 143 -[default1]:[2022-09-07 22:14:53,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-07 22:14:53,963] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 49 -[default6]:[2022-09-07 22:14:53,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-07 22:14:53,948] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 254 -[default6]:[2022-09-07 22:14:53,913] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 86 -[default7]:[2022-09-07 22:14:54,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-07 22:14:54,002] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 207 -[default2]:[2022-09-07 22:14:54,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-07 22:14:54,058] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 122 -[default0]:[2022-09-07 22:14:54,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-07 22:14:54,038] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 144 -[default4]:[2022-09-07 22:14:53,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-07 22:14:53,994] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 148 -[default2]:[2022-09-07 22:14:54,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-07 22:14:54,022] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 114 -[default4]:[2022-09-07 22:14:54,022] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 132 -[default0]:[2022-09-07 22:14:53,966] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 128 -[default2]:[2022-09-07 22:14:54,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-07 22:14:54,015] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 26 -[default7]:[2022-09-07 22:14:53,999] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 135 -[default3]:[2022-09-07 22:14:53,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-07 22:14:53,993] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 67 -[default4]:[2022-09-07 22:14:53,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-07 22:14:53,989] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 164 -[default1]:[2022-09-07 22:14:54,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-07 22:14:54,011] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 193 -[default3]:[2022-09-07 22:14:54,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-07 22:14:54,026] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 147 -[default3]:[2022-09-07 22:14:54,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-07 22:14:54,041] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 35 -[default4]:[2022-09-07 22:14:53,993] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 84 -[default5]:[2022-09-07 22:14:54,118] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 269 -[default2]:[2022-09-07 22:14:54,131] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 58 -[default5]:[2022-09-07 22:14:54,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-07 22:14:54,091] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 173 -[default1]:[2022-09-07 22:14:54,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-07 22:14:54,094] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 25 -[default5]:[2022-09-07 22:14:54,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-07 22:14:54,093] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 125 -[default1]:[2022-09-07 22:14:54,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-07 22:14:54,106] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 33 -[default7]:[2022-09-07 22:14:54,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-07 22:14:54,075] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 71 -[default5]:[2022-09-07 22:14:54,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-07 22:14:54,084] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 165 -[default5]:[2022-09-07 22:14:54,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-07 22:14:54,083] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 77 -[default6]:[2022-09-07 22:14:54,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-07 22:14:54,084] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 22 -[default2]:[2022-09-07 22:14:54,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-07 22:14:54,125] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 34 -[default4]:[2022-09-07 22:14:54,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-07 22:14:54,074] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 236 -[default0]:[2022-09-07 22:14:54,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-07 22:14:54,087] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 64 -[default7]:[2022-09-07 22:14:54,111] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 159 -[default5]:[2022-09-07 22:14:54,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-07 22:14:54,130] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 181 -[default4]:[2022-09-07 22:14:54,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-07 22:14:54,090] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 100 -[default6]:[2022-09-07 22:14:54,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-07 22:14:54,134] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 166 -[default4]:[2022-09-07 22:14:54,157] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 244 -[default6]:[2022-09-07 22:14:54,106] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 110 -[default7]:[2022-09-07 22:14:54,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-07 22:14:54,119] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 255 -[default6]:[2022-09-07 22:14:54,255] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 278 -[default0]:[2022-09-07 22:14:54,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-07 22:14:54,234] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 200 -[default1]:[2022-09-07 22:14:54,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-07 22:14:54,250] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 145 -[default6]:[2022-09-07 22:14:54,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-07 22:14:54,192] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 174 -[default3]:[2022-09-07 22:14:54,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-07 22:14:54,188] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 27 -[default4]:[2022-09-07 22:14:54,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-07 22:14:54,205] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 68 -[default6]:[2022-09-07 22:14:54,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-07 22:14:54,222] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 70 -[default4]:[2022-09-07 22:14:54,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-07 22:14:54,199] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 172 -[default4]:[2022-09-07 22:14:54,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-07 22:14:54,188] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 20 -[default5]:[2022-09-07 22:14:54,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-07 22:14:54,256] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 21 -[default7]:[2022-09-07 22:14:54,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-07 22:14:54,198] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 23 -[default1]:[2022-09-07 22:14:54,205] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 17 -[default4]:[2022-09-07 22:14:54,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-07 22:14:54,188] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 252 -[default4]:[2022-09-07 22:14:54,257] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 260 -[default6]:[2022-09-07 22:14:54,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-07 22:14:54,244] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 238 -[default2]:[2022-09-07 22:14:54,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-07 22:14:54,269] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 50 -[default7]:[2022-09-07 22:14:54,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-07 22:14:54,204] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 39 -[default7]:[2022-09-07 22:14:54,209] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 215 -[default4]:[2022-09-07 22:14:54,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-07 22:14:54,224] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 4 -[default6]:[2022-09-07 22:14:54,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-07 22:14:54,274] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 6 -[default4]:[2022-09-07 22:14:54,326] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 268 -[default6]:[2022-09-07 22:14:54,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-07 22:14:54,335] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 150 -[default7]:[2022-09-07 22:14:54,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-07 22:14:54,302] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 151 -[default1]:[2022-09-07 22:14:54,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-07 22:14:54,334] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 113 -[default6]:[2022-09-07 22:14:54,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-07 22:14:54,322] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 198 -[default7]:[2022-09-07 22:14:54,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-07 22:14:54,344] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 199 -[default4]:[2022-09-07 22:14:54,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-07 22:14:54,311] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 196 -[default4]:[2022-09-07 22:14:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-07 22:14:54,285] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 28 -[default4]:[2022-09-07 22:14:54,339] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 156 -[default7]:[2022-09-07 22:14:54,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-07 22:14:54,345] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 175 -[default0]:[2022-09-07 22:14:54,272] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 216 -[default1]:[2022-09-07 22:14:54,372] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 73 -[default5]:[2022-09-07 22:14:54,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-07 22:14:54,376] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 37 -[default0]:[2022-09-07 22:14:54,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-07 22:14:54,383] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 112 -[default2]:[2022-09-07 22:14:54,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-07 22:14:54,377] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 202 -[default5]:[2022-09-07 22:14:54,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-07 22:14:54,380] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 149 -[default5]:[2022-09-07 22:14:54,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-07 22:14:54,425] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 205 -[default4]:[2022-09-07 22:14:54,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-07 22:14:54,385] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 204 -[default6]:[2022-09-07 22:14:54,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-07 22:14:54,414] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 118 -[default5]:[2022-09-07 22:14:54,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default7]:[2022-09-07 22:14:54,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-07 22:14:54,453] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 119 -[default4]:[2022-09-07 22:14:54,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-07 22:14:54,430] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 116 -[default5]:[2022-09-07 22:14:54,427] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 117 -[default5]:[2022-09-07 22:14:54,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-07 22:14:54,377] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 197 -[default7]:[2022-09-07 22:14:54,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-07 22:14:54,411] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 31 -[default3]:[2022-09-07 22:14:54,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-07 22:14:54,418] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 203 -[default5]:[2022-09-07 22:14:54,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-07 22:14:54,394] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 237 -[default7]:[2022-09-07 22:14:54,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-07 22:14:54,460] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 239 -[default1]:[2022-09-07 22:14:54,401] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 153 -[default3]:[2022-09-07 22:14:54,471] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 251 -[default0]:[2022-09-07 22:14:54,449] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 248 -[default3]:[2022-09-07 22:14:54,385] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 51 -[default7]:[2022-09-07 22:14:54,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-07 22:14:54,404] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 7 -[default6]:[2022-09-07 22:14:54,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-07 22:14:54,421] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 54 -[default3]:[2022-09-07 22:14:54,468] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 43 -[default3]:[2022-09-07 22:14:54,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-07 22:14:54,534] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 115 -[default7]:[2022-09-07 22:14:54,538] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 47 -[default5]:[2022-09-07 22:14:54,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-07 22:14:54,492] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 29 -[default2]:[2022-09-07 22:14:54,490] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 98 -[default2]:[2022-09-07 22:14:54,545] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 218 -[default4]:[2022-09-07 22:14:54,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-07 22:14:54,500] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 52 -[default5]:[2022-09-07 22:14:54,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-07 22:14:54,521] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 53 -[default7]:[2022-09-07 22:14:54,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-07 22:14:54,506] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 55 -[default1]:[2022-09-07 22:14:54,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-07 22:14:54,586] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 201 -[default2]:[2022-09-07 22:14:54,660] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 194 -[default3]:[2022-09-07 22:14:54,650] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 19 -[default0]:[2022-09-07 22:14:54,696] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 120 -[default4]:[2022-09-07 22:14:54,704] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 44 -[default5]:[2022-09-07 22:14:54,677] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 261 -[default1]:[2022-09-07 22:14:54,793] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 41 -[default0]:[2022-09-07 22:14:54,844] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 40 -[default5]:[2022-09-07 22:14:54,852] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 277 -[default2]:[2022-09-07 22:14:54,792] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 234 -[default7]:[2022-09-07 22:14:54,798] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 111 -[default0]:[2022-09-07 22:14:54,882] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 48 -[default4]:[2022-09-07 22:14:54,881] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 108 -[default4]:[2022-09-07 22:14:54,955] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 276 -[default4]:[2022-09-07 22:14:54,963] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 60 -[default0]:[2022-09-07 22:14:54,884] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 160 -[default3]:[2022-09-07 22:14:54,947] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 163 -[default5]:[2022-09-07 22:14:54,939] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 45 -[default7]:[2022-09-07 22:14:54,881] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 223 -[default5]:[2022-09-07 22:14:54,891] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 245 -[default6]:[2022-09-07 22:14:54,899] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 222 -[default6]:[2022-09-07 22:14:54,914] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 246 -[default5]:[2022-09-07 22:14:54,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-07 22:14:54,925] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 5 -[default7]:[2022-09-07 22:14:55,022] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 63 -[default6]:[2022-09-07 22:14:55,047] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 62 -[default5]:[2022-09-07 22:14:55,008] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 61 -[default2]:[2022-09-07 22:14:55,053] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 170 -[default2]:[2022-09-07 22:14:55,048] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 18 -[default3]:[2022-09-07 22:14:55,014] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 219 -[default3]:[2022-09-07 22:14:55,112] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 171 -[default6]:[2022-09-07 22:14:55,134] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 30 -[default1]:[2022-09-07 22:14:55,081] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 249 -[default7]:[2022-09-07 22:14:55,103] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 247 -[default0]:[2022-09-07 22:14:55,193] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 176 -[default1]:[2022-09-07 22:14:55,193] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 49 -[default3]:[2022-09-07 22:14:55,358] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 179 -[default6]:[2022-09-07 22:14:55,339] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 158 -[default3]:[2022-09-07 22:14:55,312] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 123 -[default0]:[2022-09-07 22:14:55,409] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 168 -[default1]:[2022-09-07 22:14:55,424] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 169 -[default0]:[2022-09-07 22:14:55,400] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 152 -[default2]:[2022-09-07 22:14:55,545] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 154 -[default3]:[2022-09-07 22:14:55,484] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 155 -[default3]:[2022-09-07 22:14:55,632] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 195 -[default2]:[2022-09-07 22:14:55,584] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 74 -[default2]:[2022-09-07 22:14:55,609] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 138 -[default7]:[2022-09-07 22:14:55,762] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 207 -[default1]:[2022-09-07 22:14:55,686] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 121 -[default5]:[2022-09-07 22:14:55,670] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 69 -[default1]:[2022-09-07 22:14:55,759] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 177 -[default1]:[2022-09-07 22:14:55,752] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 193 -[default7]:[2022-09-07 22:14:55,682] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 79 -[default2]:[2022-09-07 22:14:55,828] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 114 -[default2]:[2022-09-07 22:14:55,861] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 66 -[default1]:[2022-09-07 22:14:55,853] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 161 -[default0]:[2022-09-07 22:14:55,853] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 96 -[default2]:[2022-09-07 22:14:55,845] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 50 -[default6]:[2022-09-07 22:14:55,849] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 142 -[default6]:[2022-09-07 22:14:55,937] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 78 -[default1]:[2022-09-07 22:14:55,910] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 65 -[default6]:[2022-09-07 22:14:55,940] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 254 -[default7]:[2022-09-07 22:14:55,959] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 255 -[default6]:[2022-09-07 22:14:56,002] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 206 -[default3]:[2022-09-07 22:14:55,991] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 67 -[default2]:[2022-09-07 22:14:56,070] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 178 -[default3]:[2022-09-07 22:14:56,021] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 99 -[default1]:[2022-09-07 22:14:56,027] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 233 -[default4]:[2022-09-07 22:14:56,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-07 22:14:56,024] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 284 -[default5]:[2022-09-07 22:14:55,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-07 22:14:55,991] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 285 -[default3]:[2022-09-07 22:14:55,999] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 139 -[default1]:[2022-09-07 22:14:56,052] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 97 -[default2]:[2022-09-07 22:14:56,071] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 26 -[default2]:[2022-09-07 22:14:56,085] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 162 -[default6]:[2022-09-07 22:14:56,148] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 22 -[default0]:[2022-09-07 22:14:56,150] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 64 -[default5]:[2022-09-07 22:14:56,170] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 253 -[default7]:[2022-09-07 22:14:56,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-07 22:14:56,177] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 287 -[default2]:[2022-09-07 22:14:56,193] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 202 -[default0]:[2022-09-07 22:14:56,200] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 200 -[default2]:[2022-09-07 22:14:56,260] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 146 -[default1]:[2022-09-07 22:14:56,221] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 113 -[default7]:[2022-09-07 22:14:56,182] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 23 -[default4]:[2022-09-07 22:14:56,222] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 76 -[default4]:[2022-09-07 22:14:56,224] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 252 -[default3]:[2022-09-07 22:14:56,182] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 235 -[default2]:[2022-09-07 22:14:56,356] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 122 -[default1]:[2022-09-07 22:14:56,324] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 25 -[default1]:[2022-09-07 22:14:56,284] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 201 -[default4]:[2022-09-07 22:14:56,311] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 68 -[default7]:[2022-09-07 22:14:56,358] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 71 -[default4]:[2022-09-07 22:14:56,277] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 20 -[default7]:[2022-09-07 22:14:56,341] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 167 -[default5]:[2022-09-07 22:14:56,293] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 77 -[default5]:[2022-09-07 22:14:56,298] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 21 -[default3]:[2022-09-07 22:14:56,316] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 147 -[default0]:[2022-09-07 22:14:56,312] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 32 -[default6]:[2022-09-07 22:14:56,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-07 22:14:56,291] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 286 -[default1]:[2022-09-07 22:14:56,418] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 33 -[default3]:[2022-09-07 22:14:56,412] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 203 -[default0]:[2022-09-07 22:14:56,410] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 232 -[default3]:[2022-09-07 22:14:56,463] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 35 -[default5]:[2022-09-07 22:14:56,380] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 141 -[default7]:[2022-09-07 22:14:56,420] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 183 -[default6]:[2022-09-07 22:14:56,481] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 174 -[default6]:[2022-09-07 22:14:56,528] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 70 -[default7]:[2022-09-07 22:14:56,513] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 175 -[default4]:[2022-09-07 22:14:56,549] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 140 -[default7]:[2022-09-07 22:14:56,500] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 143 -[default5]:[2022-09-07 22:14:56,579] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 173 -[default5]:[2022-09-07 22:14:56,606] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 205 -[default4]:[2022-09-07 22:14:56,584] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 204 -[default4]:[2022-09-07 22:14:56,601] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 172 -[default6]:[2022-09-07 22:14:56,637] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 102 -[default6]:[2022-09-07 22:14:56,620] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 182 -[default7]:[2022-09-07 22:14:56,662] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 55 -[default6]:[2022-09-07 22:14:56,639] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 54 -[default2]:[2022-09-07 22:14:56,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-07 22:14:56,688] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 2 -[default7]:[2022-09-07 22:14:56,684] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 127 -[default6]:[2022-09-07 22:14:56,705] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 198 -[default7]:[2022-09-07 22:14:56,738] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 103 -[default4]:[2022-09-07 22:14:56,750] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 180 -[default4]:[2022-09-07 22:14:56,757] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 4 -[default3]:[2022-09-07 22:14:56,859] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 115 -[default6]:[2022-09-07 22:14:56,778] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 126 -[default7]:[2022-09-07 22:14:56,802] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 199 -[default5]:[2022-09-07 22:14:56,848] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 101 -[default5]:[2022-09-07 22:14:56,834] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 181 -[default4]:[2022-09-07 22:14:56,865] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 52 -[default5]:[2022-09-07 22:14:56,868] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 53 -[default5]:[2022-09-07 22:14:56,935] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 125 -[default3]:[2022-09-07 22:14:56,883] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 27 -[default7]:[2022-09-07 22:14:56,930] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 31 -[default4]:[2022-09-07 22:14:56,914] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 124 -[default4]:[2022-09-07 22:14:56,932] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 100 -[default0]:[2022-09-07 22:14:57,065] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 144 -[default5]:[2022-09-07 22:14:57,052] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 197 -[default4]:[2022-09-07 22:14:56,997] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 196 -[default4]:[2022-09-07 22:14:57,127] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 164 -[default5]:[2022-09-07 22:14:57,143] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 165 -[default2]:[2022-09-07 22:14:57,111] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 34 -[default4]:[2022-09-07 22:14:57,151] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 236 -[default6]:[2022-09-07 22:14:57,114] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 238 -[default6]:[2022-09-07 22:14:57,103] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 38 -[default7]:[2022-09-07 22:14:57,139] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 239 -[default6]:[2022-09-07 22:14:57,146] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 166 -[default0]:[2022-09-07 22:14:57,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-07 22:14:57,126] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 0 -[default1]:[2022-09-07 22:14:57,193] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 145 -[default4]:[2022-09-07 22:14:57,252] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 28 -[default5]:[2022-09-07 22:14:57,237] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 237 -[default7]:[2022-09-07 22:14:57,311] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 119 -[default5]:[2022-09-07 22:14:57,273] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 29 -[default7]:[2022-09-07 22:14:57,320] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 39 -[default3]:[2022-09-07 22:14:57,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-07 22:14:57,314] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 3 -[default5]:[2022-09-07 22:14:57,370] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 5 -[default6]:[2022-09-07 22:14:57,335] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 6 -[default0]:[2022-09-07 22:14:57,394] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 112 -[default6]:[2022-09-07 22:14:57,423] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 150 -[default6]:[2022-09-07 22:14:57,432] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 118 -[default4]:[2022-09-07 22:14:57,386] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 36 -[default1]:[2022-09-07 22:14:57,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step5/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-07 22:14:57,474] [INFO] [engine.py:2833:_get_all_zero_checkpoint_state_dicts] successfully read 4 ZeRO state_dicts for rank 1 -[default4]:[2022-09-07 22:14:57,501] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 148 -[default7]:[2022-09-07 22:14:57,489] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 151 -[default5]:[2022-09-07 22:14:57,522] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 37 -[default7]:[2022-09-07 22:14:57,501] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 7 -[default5]:[2022-09-07 22:14:57,649] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 149 -[default4]:[2022-09-07 22:14:57,754] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 116 -[default5]:[2022-09-07 22:14:57,754] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 117 -[default5]:[2022-09-07 22:14:58,259] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 285 -[default4]:[2022-09-07 22:14:58,245] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 284 -[default2]:[2022-09-07 22:14:58,346] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 2 -[default0]:[2022-09-07 22:14:58,667] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 0 -[default0]: checkpoint version 3.0 -[default6]:[2022-09-07 22:14:58,761] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 286 -[default7]:[2022-09-07 22:14:58,693] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 287 -[default3]:[2022-09-07 22:14:58,969] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 3 -[default1]:[2022-09-07 22:14:59,984] [INFO] [engine.py:2767:_load_zero_checkpoint] loading 4 zero partition checkpoints for rank 1 -[default0]: successfully loaded checkpoint from /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq at iteration 5 -[default0]:estimated model parameters: 258.958393344 -[default0]:estimated model parameters without embeddings: 0.002064384 -[default0]:[after model, optimizer, and learning rate scheduler are built] datetime: 2022-09-07 22:15:00 -[default0]:> building train, validation, and test datasets ... -[default0]: > datasets target sizes (minimum size): -[default0]: train: 6348800 -[default0]: validation: 26624 -[default0]: test: 2048 -[default0]:> building train, validation, and test datasets for T0 ... -[default0]: > building dataset index ... -[default7]:time (ms) | load-checkpoint: 23505.16 -[default0]:/gpfsssd/worksf/projects/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/megatron/utils.py:365: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings -[default0]: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.259591 seconds -[default0]: number of documents: 32740750 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 32740750) total of 32740750 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.006546 seconds -[default0]: number of documents: 32740750 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003381 seconds -[default0]: number of documents: 32740750 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_en_train_indexmap_2470768ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_en_train_indexmap_2470768ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.143 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.037650 seconds -[default0]: number of documents: 5413205 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 5413205) total of 5413205 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.138075 seconds -[default0]: number of documents: 5413205 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003071 seconds -[default0]: number of documents: 5413205 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_es_train_indexmap_540571ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_es_train_indexmap_540571ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.081 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.118176 seconds -[default0]: number of documents: 3752156 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 3752156) total of 3752156 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.078820 seconds -[default0]: number of documents: 3752156 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003454 seconds -[default0]: number of documents: 3752156 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pt_train_indexmap_433493ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pt_train_indexmap_433493ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.168 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.088997 seconds -[default0]: number of documents: 5316403 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 5316403) total of 5316403 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.052382 seconds -[default0]: number of documents: 5316403 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003555 seconds -[default0]: number of documents: 5316403 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fr_train_indexmap_414881ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fr_train_indexmap_414881ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.191 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.118725 seconds -[default0]: number of documents: 2707724 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 2707724) total of 2707724 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.267898 seconds -[default0]: number of documents: 2707724 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003760 seconds -[default0]: number of documents: 2707724 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_code_train_indexmap_370738ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_code_train_indexmap_370738ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.191 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.061797 seconds -[default0]: number of documents: 2160181 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 2160181) total of 2160181 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.014394 seconds -[default0]: number of documents: 2160181 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.002642 seconds -[default0]: number of documents: 2160181 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ar_train_indexmap_294803ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ar_train_indexmap_294803ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.082 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.096251 seconds -[default0]: number of documents: 2643418 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 2643418) total of 2643418 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.040835 seconds -[default0]: number of documents: 2643418 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003944 seconds -[default0]: number of documents: 2643418 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_id_train_indexmap_290321ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_id_train_indexmap_290321ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.174 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.097047 seconds -[default0]: number of documents: 3589234 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 3589234) total of 3589234 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.018843 seconds -[default0]: number of documents: 3589234 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004117 seconds -[default0]: number of documents: 3589234 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zh_train_indexmap_289514ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zh_train_indexmap_289514ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.258 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.062638 seconds -[default0]: number of documents: 1554667 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 1554667) total of 1554667 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.050467 seconds -[default0]: number of documents: 1554667 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003778 seconds -[default0]: number of documents: 1554667 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_hi_train_indexmap_277185ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_hi_train_indexmap_277185ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.106 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.062389 seconds -[default0]: number of documents: 1672106 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 1672106) total of 1672106 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.204367 seconds -[default0]: number of documents: 1672106 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003689 seconds -[default0]: number of documents: 1672106 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_vi_train_indexmap_195289ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_vi_train_indexmap_195289ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.113 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.064441 seconds -[default0]: number of documents: 855756 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 855756) total of 855756 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.078175 seconds -[default0]: number of documents: 855756 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004164 seconds -[default0]: number of documents: 855756 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ur_train_indexmap_120747ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ur_train_indexmap_120747ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.164 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.160150 seconds -[default0]: number of documents: 584590 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 584590) total of 584590 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.072261 seconds -[default0]: number of documents: 584590 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.002539 seconds -[default0]: number of documents: 584590 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_te_train_indexmap_84551ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_te_train_indexmap_84551ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.092 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.066133 seconds -[default0]: number of documents: 415433 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 415433) total of 415433 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.042001 seconds -[default0]: number of documents: 415433 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001576 seconds -[default0]: number of documents: 415433 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ta_train_indexmap_58345ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ta_train_indexmap_58345ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.137 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.067256 seconds -[default0]: number of documents: 428843 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 428843) total of 428843 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.056059 seconds -[default0]: number of documents: 428843 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001770 seconds -[default0]: number of documents: 428843 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bn_train_indexmap_52416ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bn_train_indexmap_52416ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.153 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.042201 seconds -[default0]: number of documents: 417269 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 417269) total of 417269 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.123674 seconds -[default0]: number of documents: 417269 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001826 seconds -[default0]: number of documents: 417269 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_mr_train_indexmap_41937ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_mr_train_indexmap_41937ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.082 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.062077 seconds -[default0]: number of documents: 1130481 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 1130481) total of 1130481 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.051330 seconds -[default0]: number of documents: 1130481 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003388 seconds -[default0]: number of documents: 1130481 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sw_train_indexmap_35669ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sw_train_indexmap_35669ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.060 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.052692 seconds -[default0]: number of documents: 347499 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 347499) total of 347499 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.080576 seconds -[default0]: number of documents: 347499 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000953 seconds -[default0]: number of documents: 347499 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_gu_train_indexmap_35293ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_gu_train_indexmap_35293ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.056 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.092564 seconds -[default0]: number of documents: 339210 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 339210) total of 339210 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.057740 seconds -[default0]: number of documents: 339210 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001032 seconds -[default0]: number of documents: 339210 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pa_train_indexmap_32937ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_pa_train_indexmap_32937ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.119 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.026594 seconds -[default0]: number of documents: 315754 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 315754) total of 315754 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.087766 seconds -[default0]: number of documents: 315754 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001127 seconds -[default0]: number of documents: 315754 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ne_train_indexmap_24781ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ne_train_indexmap_24781ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.098 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.074945 seconds -[default0]: number of documents: 918416 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 918416) total of 918416 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.071857 seconds -[default0]: number of documents: 918416 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.005466 seconds -[default0]: number of documents: 918416 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_yo_train_indexmap_22207ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_yo_train_indexmap_22207ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.079 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.160934 seconds -[default0]: number of documents: 950097 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 950097) total of 950097 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.043889 seconds -[default0]: number of documents: 950097 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004914 seconds -[default0]: number of documents: 950097 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ig_train_indexmap_20472ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ig_train_indexmap_20472ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.125 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.092650 seconds -[default0]: number of documents: 915063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915063) total of 915063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.109288 seconds -[default0]: number of documents: 915063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003130 seconds -[default0]: number of documents: 915063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ny_train_indexmap_17130ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ny_train_indexmap_17130ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.087 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.074048 seconds -[default0]: number of documents: 915061 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915061) total of 915061 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.248773 seconds -[default0]: number of documents: 915061 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003221 seconds -[default0]: number of documents: 915061 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zu_train_indexmap_16600ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_zu_train_indexmap_16600ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.076 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.223314 seconds -[default0]: number of documents: 915058 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915058) total of 915058 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.050927 seconds -[default0]: number of documents: 915058 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.002931 seconds -[default0]: number of documents: 915058 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_xh_train_indexmap_16031ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_xh_train_indexmap_16031ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.091 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.057242 seconds -[default0]: number of documents: 865056 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 865056) total of 865056 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.064480 seconds -[default0]: number of documents: 865056 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.002781 seconds -[default0]: number of documents: 865056 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sn_train_indexmap_15894ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_sn_train_indexmap_15894ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.071 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.164765 seconds -[default0]: number of documents: 915044 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915044) total of 915044 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.093046 seconds -[default0]: number of documents: 915044 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003152 seconds -[default0]: number of documents: 915044 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ts_train_indexmap_15753ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ts_train_indexmap_15753ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.067 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.113867 seconds -[default0]: number of documents: 915043 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915043) total of 915043 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.055721 seconds -[default0]: number of documents: 915043 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.002833 seconds -[default0]: number of documents: 915043 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rw_train_indexmap_15697ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rw_train_indexmap_15697ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.090 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.110438 seconds -[default0]: number of documents: 915021 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915021) total of 915021 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.163845 seconds -[default0]: number of documents: 915021 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.003211 seconds -[default0]: number of documents: 915021 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_lg_train_indexmap_14852ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_lg_train_indexmap_14852ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.079 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.070849 seconds -[default0]: number of documents: 915054 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915054) total of 915054 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.107561 seconds -[default0]: number of documents: 915054 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.004242 seconds -[default0]: number of documents: 915054 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tn_train_indexmap_14826ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tn_train_indexmap_14826ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.045 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.061241 seconds -[default0]: number of documents: 915051 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 915051) total of 915051 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.068409 seconds -[default0]: number of documents: 915051 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.002950 seconds -[default0]: number of documents: 915051 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_nso_train_indexmap_14460ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_nso_train_indexmap_14460ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.055 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.109248 seconds -[default0]: number of documents: 318189 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 318189) total of 318189 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.026966 seconds -[default0]: number of documents: 318189 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001189 seconds -[default0]: number of documents: 318189 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rn_train_indexmap_12148ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_rn_train_indexmap_12148ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.048 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.025122 seconds -[default0]: number of documents: 265864 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265864) total of 265864 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.031501 seconds -[default0]: number of documents: 265864 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000986 seconds -[default0]: number of documents: 265864 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ml_train_indexmap_11018ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ml_train_indexmap_11018ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.029 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.054527 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.089789 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000838 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_kn_train_indexmap_10415ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_kn_train_indexmap_10415ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.035 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.076512 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.147491 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000986 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_or_train_indexmap_10164ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_or_train_indexmap_10164ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.026 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.036391 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.072222 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000963 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_as_train_indexmap_9836ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_as_train_indexmap_9836ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.044 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.048617 seconds -[default0]: number of documents: 365060 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 365060) total of 365060 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.041410 seconds -[default0]: number of documents: 365060 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000953 seconds -[default0]: number of documents: 365060 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ln_train_indexmap_7951ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ln_train_indexmap_7951ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.040 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.055209 seconds -[default0]: number of documents: 365063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 365063) total of 365063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.034064 seconds -[default0]: number of documents: 365063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000905 seconds -[default0]: number of documents: 365063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_wo_train_indexmap_7715ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_wo_train_indexmap_7715ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.061 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.044766 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.045877 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000699 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tum_train_indexmap_7304ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tum_train_indexmap_7304ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.055 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.041692 seconds -[default0]: number of documents: 265180 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265180) total of 265180 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.105091 seconds -[default0]: number of documents: 265180 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000911 seconds -[default0]: number of documents: 265180 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ki_train_indexmap_7242ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ki_train_indexmap_7242ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.037 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.080752 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.046608 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000811 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_st_train_indexmap_7181ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_st_train_indexmap_7181ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.039 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.079444 seconds -[default0]: number of documents: 265063 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265063) total of 265063 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.160422 seconds -[default0]: number of documents: 265063 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000842 seconds -[default0]: number of documents: 265063 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fon_train_indexmap_7118ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_fon_train_indexmap_7118ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.027 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.049042 seconds -[default0]: number of documents: 281199 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 281199) total of 281199 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.033108 seconds -[default0]: number of documents: 281199 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000872 seconds -[default0]: number of documents: 281199 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_eu_train_indexmap_7114ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_eu_train_indexmap_7114ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.038 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.047884 seconds -[default0]: number of documents: 271191 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 271191) total of 271191 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.090993 seconds -[default0]: number of documents: 271191 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000839 seconds -[default0]: number of documents: 271191 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ca_train_indexmap_6963ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ca_train_indexmap_6963ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.032 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.048404 seconds -[default0]: number of documents: 265071 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265071) total of 265071 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.037156 seconds -[default0]: number of documents: 265071 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.001093 seconds -[default0]: number of documents: 265071 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ak_train_indexmap_6805ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_ak_train_indexmap_6805ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.055 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.028621 seconds -[default0]: number of documents: 265180 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265180) total of 265180 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.064486 seconds -[default0]: number of documents: 265180 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000841 seconds -[default0]: number of documents: 265180 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bm_train_indexmap_6739ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_bm_train_indexmap_6739ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.033 seconds -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.094037 seconds -[default0]: number of documents: 265071 -[default0]: > dataset split: -[default0]: train: -[default0]: document indices in [0, 265071) total of 265071 documents -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.040826 seconds -[default0]: number of documents: 265071 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.000781 seconds -[default0]: number of documents: 265071 -[default0]: > loading doc-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tw_train_indexmap_6691ns_42s_decoder_packed_batch_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfswork/rech/six/commun/bigscience-training/xp3zz/xp3_tw_train_indexmap_6691ns_42s_decoder_packed_shuffle_idx.npy -[default0]: loaded indexed file in 0.080 seconds -[default0]:> building indices for blendable datasets ... -[default0]: > sample ratios: -[default0]: dataset 0, input: 0.387235, achieved: 0.387235 -[default0]: dataset 1, input: 0.0847217, achieved: 0.0847217 -[default0]: dataset 2, input: 0.0679398, achieved: 0.0679398 -[default0]: dataset 3, input: 0.0650228, achieved: 0.0650228 -[default0]: dataset 4, input: 0.0581044, achieved: 0.0581043 -[default0]: dataset 5, input: 0.0462033, achieved: 0.0462033 -[default0]: dataset 6, input: 0.0455009, achieved: 0.0455009 -[default0]: dataset 7, input: 0.0453745, achieved: 0.0453744 -[default0]: dataset 8, input: 0.0434421, achieved: 0.0434421 -[default0]: dataset 9, input: 0.030607, achieved: 0.030607 -[default0]: dataset 10, input: 0.0189242, achieved: 0.0189242 -[default0]: dataset 11, input: 0.0132513, achieved: 0.0132513 -[default0]: dataset 12, input: 0.00914419, achieved: 0.00914417 -[default0]: dataset 13, input: 0.00821496, achieved: 0.00821492 -[default0]: dataset 14, input: 0.0065726, achieved: 0.00657258 -[default0]: dataset 15, input: 0.00559018, achieved: 0.00559023 -[default0]: dataset 16, input: 0.00553131, achieved: 0.00553135 -[default0]: dataset 17, input: 0.00516195, achieved: 0.00516192 -[default0]: dataset 18, input: 0.00388374, achieved: 0.00388376 -[default0]: dataset 19, input: 0.00348029, achieved: 0.00348033 -[default0]: dataset 20, input: 0.00320848, achieved: 0.0032085 -[default0]: dataset 21, input: 0.0026846, achieved: 0.00268464 -[default0]: dataset 22, input: 0.00260158, achieved: 0.00260161 -[default0]: dataset 23, input: 0.00251239, achieved: 0.00251236 -[default0]: dataset 24, input: 0.00249093, achieved: 0.00249096 -[default0]: dataset 25, input: 0.00246883, achieved: 0.00246885 -[default0]: dataset 26, input: 0.00245999, achieved: 0.00245997 -[default0]: dataset 27, input: 0.00232756, achieved: 0.00232756 -[default0]: dataset 28, input: 0.00232361, achieved: 0.00232365 -[default0]: dataset 29, input: 0.00226616, achieved: 0.00226619 -[default0]: dataset 30, input: 0.00190391, achieved: 0.00190387 -[default0]: dataset 31, input: 0.00172681, achieved: 0.0017268 -[default0]: dataset 32, input: 0.00163226, achieved: 0.00163222 -[default0]: dataset 33, input: 0.00159296, achieved: 0.00159297 -[default0]: dataset 34, input: 0.0015415, achieved: 0.00154146 -[default0]: dataset 35, input: 0.00124602, achieved: 0.00124601 -[default0]: dataset 36, input: 0.00120908, achieved: 0.00120907 -[default0]: dataset 37, input: 0.00114468, achieved: 0.00114469 -[default0]: dataset 38, input: 0.00113489, achieved: 0.00113492 -[default0]: dataset 39, input: 0.00112542, achieved: 0.00112542 -[default0]: dataset 40, input: 0.00111548, achieved: 0.00111547 -[default0]: dataset 41, input: 0.00111485, achieved: 0.00111485 -[default0]: dataset 42, input: 0.00109117, achieved: 0.00109114 -[default0]: dataset 43, input: 0.00106639, achieved: 0.00106636 -[default0]: dataset 44, input: 0.00105613, achieved: 0.00105615 -[default0]: dataset 45, input: 0.00104855, achieved: 0.0010486 -[default0]:> elapsed time for building blendable dataset indices: 0.57 (sec) -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.182708 seconds -[default0]: number of documents: 15234080 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [14472376, 15234080) total of 761704 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ar/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ar_text_document_validation_pretraining_indexmap_885ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ar/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ar_text_document_validation_pretraining_indexmap_885ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ar/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ar_text_document_validation_pretraining_indexmap_885ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.253 seconds -[default0]: total number of samples: 221750 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.249306 seconds -[default0]: number of documents: 6142390 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [5835270, 6142390) total of 307120 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ca/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ca_text_document_validation_pretraining_indexmap_301ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ca/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ca_text_document_validation_pretraining_indexmap_301ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/ca/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_ca_text_document_validation_pretraining_indexmap_301ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.066 seconds -[default0]: total number of samples: 136143 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.092729 seconds -[default0]: number of documents: 26176998 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [24868148, 26176998) total of 1308850 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/code/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_code_text_document_validation_pretraining_indexmap_3486ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/code/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_code_text_document_validation_pretraining_indexmap_3486ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/code/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_code_text_document_validation_pretraining_indexmap_3486ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.143 seconds -[default0]: total number of samples: 432311 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.368361 seconds -[default0]: number of documents: 20844665 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [19802432, 20844665) total of 1042233 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/en/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_en_text_document_validation_pretraining_indexmap_5933ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/en/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_en_text_document_validation_pretraining_indexmap_5933ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/en/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_en_text_document_validation_pretraining_indexmap_5933ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.141 seconds -[default0]: total number of samples: 521545 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.259203 seconds -[default0]: number of documents: 67005817 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [63655526, 67005817) total of 3350291 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/es/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_es_text_document_validation_pretraining_indexmap_2855ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/es/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_es_text_document_validation_pretraining_indexmap_2855ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/es/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_es_text_document_validation_pretraining_indexmap_2855ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.232 seconds -[default0]: total number of samples: 1740321 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.247258 seconds -[default0]: number of documents: 5149795 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [4892305, 5149795) total of 257490 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/eu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_eu_text_document_validation_pretraining_indexmap_42ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/eu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_eu_text_document_validation_pretraining_indexmap_42ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/eu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_eu_text_document_validation_pretraining_indexmap_42ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.053 seconds -[default0]: total number of samples: 26370 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.131662 seconds -[default0]: number of documents: 58847091 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [55904736, 58847091) total of 2942355 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/fr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_fr_text_document_validation_pretraining_indexmap_3493ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/fr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_fr_text_document_validation_pretraining_indexmap_3493ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/fr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_fr_text_document_validation_pretraining_indexmap_3493ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.206 seconds -[default0]: total number of samples: 1458654 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.279788 seconds -[default0]: number of documents: 12514253 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [11888540, 12514253) total of 625713 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/id/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_id_text_document_validation_pretraining_indexmap_293ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/id/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_id_text_document_validation_pretraining_indexmap_293ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/id/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_id_text_document_validation_pretraining_indexmap_293ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.091 seconds -[default0]: total number of samples: 134071 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.106301 seconds -[default0]: number of documents: 180608 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [171578, 180608) total of 9030 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-as/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-as_text_document_validation_pretraining_indexmap_3ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-as/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-as_text_document_validation_pretraining_indexmap_3ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-as/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-as_text_document_validation_pretraining_indexmap_3ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.009 seconds -[default0]: total number of samples: 2501 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.169490 seconds -[default0]: number of documents: 12303134 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [11687977, 12303134) total of 615157 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-bn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-bn_text_document_validation_pretraining_indexmap_147ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-bn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-bn_text_document_validation_pretraining_indexmap_147ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-bn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-bn_text_document_validation_pretraining_indexmap_147ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.121 seconds -[default0]: total number of samples: 157244 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.229141 seconds -[default0]: number of documents: 2033057 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [1931404, 2033057) total of 101653 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-gu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-gu_text_document_validation_pretraining_indexmap_11ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-gu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-gu_text_document_validation_pretraining_indexmap_11ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-gu/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-gu_text_document_validation_pretraining_indexmap_11ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.025 seconds -[default0]: total number of samples: 20517 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.093656 seconds -[default0]: number of documents: 26793553 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [25453875, 26793553) total of 1339678 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-hi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-hi_text_document_validation_pretraining_indexmap_200ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-hi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-hi_text_document_validation_pretraining_indexmap_200ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-hi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-hi_text_document_validation_pretraining_indexmap_200ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.171 seconds -[default0]: total number of samples: 101502 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.056966 seconds -[default0]: number of documents: 3155990 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [2998190, 3155990) total of 157800 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-kn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-kn_text_document_validation_pretraining_indexmap_17ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-kn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-kn_text_document_validation_pretraining_indexmap_17ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-kn/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-kn_text_document_validation_pretraining_indexmap_17ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.050 seconds -[default0]: total number of samples: 44182 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.203807 seconds -[default0]: number of documents: 6692522 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [6357896, 6692522) total of 334626 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ml/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ml_text_document_validation_pretraining_indexmap_28ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ml/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ml_text_document_validation_pretraining_indexmap_28ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ml/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ml_text_document_validation_pretraining_indexmap_28ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.085 seconds -[default0]: total number of samples: 47613 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.208850 seconds -[default0]: number of documents: 3017261 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [2866398, 3017261) total of 150863 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-mr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-mr_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-mr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-mr_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-mr/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-mr_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.040 seconds -[default0]: total number of samples: 29298 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.277845 seconds -[default0]: number of documents: 3648041 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [3465639, 3648041) total of 182402 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ne/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ne_text_document_validation_pretraining_indexmap_18ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ne/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ne_text_document_validation_pretraining_indexmap_18ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ne/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ne_text_document_validation_pretraining_indexmap_18ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.044 seconds -[default0]: total number of samples: 5659 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.228954 seconds -[default0]: number of documents: 4327282 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [4110918, 4327282) total of 216364 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-or/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-or_text_document_validation_pretraining_indexmap_10ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-or/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-or_text_document_validation_pretraining_indexmap_10ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-or/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-or_text_document_validation_pretraining_indexmap_10ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.066 seconds -[default0]: total number of samples: 12423 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.093303 seconds -[default0]: number of documents: 2698896 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [2563951, 2698896) total of 134945 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-pa/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-pa_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-pa/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-pa_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-pa/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-pa_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.022 seconds -[default0]: total number of samples: 19133 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.113014 seconds -[default0]: number of documents: 12767593 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [12129213, 12767593) total of 638380 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ta/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ta_text_document_validation_pretraining_indexmap_57ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ta/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ta_text_document_validation_pretraining_indexmap_57ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ta/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ta_text_document_validation_pretraining_indexmap_57ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.093 seconds -[default0]: total number of samples: 87928 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.110065 seconds -[default0]: number of documents: 4342323 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [4125207, 4342323) total of 217116 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-te/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-te_text_document_validation_pretraining_indexmap_25ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-te/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-te_text_document_validation_pretraining_indexmap_25ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-te/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-te_text_document_validation_pretraining_indexmap_25ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.043 seconds -[default0]: total number of samples: 69780 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.302635 seconds -[default0]: number of documents: 3022722 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [2871586, 3022722) total of 151136 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ur/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ur_text_document_validation_pretraining_indexmap_34ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ur/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ur_text_document_validation_pretraining_indexmap_34ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/indic-ur/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_indic-ur_text_document_validation_pretraining_indexmap_34ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.044 seconds -[default0]: total number of samples: 22532 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.099442 seconds -[default0]: number of documents: 1162568 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [1104440, 1162568) total of 58128 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/nigercongo-all/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_nigercongo-all_text_document_validation_pretraining_indexmap_9ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/nigercongo-all/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_nigercongo-all_text_document_validation_pretraining_indexmap_9ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/nigercongo-all/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_nigercongo-all_text_document_validation_pretraining_indexmap_9ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.007 seconds -[default0]: total number of samples: 1608 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.140899 seconds -[default0]: number of documents: 55294645 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [52529913, 55294645) total of 2764732 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-en/meg_ds_bigscience_tokenizer_text_document_validation_pretraining_indexmap_2178ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-en/meg_ds_bigscience_tokenizer_text_document_validation_pretraining_indexmap_2178ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-en/meg_ds_bigscience_tokenizer_text_document_validation_pretraining_indexmap_2178ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.124 seconds -[default0]: total number of samples: 690621 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.237213 seconds -[default0]: number of documents: 44855616 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [42612835, 44855616) total of 2242781 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-zh/meg_ds_bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_text_document_validation_pretraining_indexmap_1480ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-zh/meg_ds_bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_text_document_validation_pretraining_indexmap_1480ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/oscar-zh/meg_ds_bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_text_document_validation_pretraining_indexmap_1480ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.158 seconds -[default0]: total number of samples: 468689 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.108770 seconds -[default0]: number of documents: 31969891 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [30371396, 31969891) total of 1598495 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/pt/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_pt_text_document_validation_pretraining_indexmap_1326ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/pt/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_pt_text_document_validation_pretraining_indexmap_1326ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/pt/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_pt_text_document_validation_pretraining_indexmap_1326ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.142 seconds -[default0]: total number of samples: 497625 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.225534 seconds -[default0]: number of documents: 34110375 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [32404856, 34110375) total of 1705519 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/vi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_vi_text_document_validation_pretraining_indexmap_659ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/vi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_vi_text_document_validation_pretraining_indexmap_659ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/vi/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_vi_text_document_validation_pretraining_indexmap_659ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.244 seconds -[default0]: total number of samples: 125120 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.049924 seconds -[default0]: number of documents: 43761623 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [41573542, 43761623) total of 2188081 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zhs/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zhs_text_document_validation_pretraining_indexmap_3236ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zhs/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zhs_text_document_validation_pretraining_indexmap_3236ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zhs/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zhs_text_document_validation_pretraining_indexmap_3236ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.159 seconds -[default0]: total number of samples: 1010592 -[default0]: total number of epochs: 1 -[default0]: > building dataset index ... -[default0]: reading sizes... -[default0]: reading pointers... -[default0]: reading document index... -[default0]: creating numpy buffer of mmap... -[default0]: creating memory view of numpy buffer... -[default0]: > finished creating indexed dataset in 0.055412 seconds -[default0]: number of documents: 197602 -[default0]: > dataset split: -[default0]: validation_pretraining: -[default0]: document indices in [187722, 197602) total of 9880 documents -[default0]: > loading doc-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zht/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zht_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_doc_idx.npy -[default0]: > loading sample-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zht/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zht_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_sample_idx.npy -[default0]: > loading shuffle-idx mapping from /gpfsdsstore/projects/rech/six/commun/merged-meg-ds_v3_pii/zht/bigscience-catalogue-data-dev_byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles_zht_text_document_validation_pretraining_indexmap_14ns_2048sl_42s_shuffle_idx.npy -[default0]: loaded indexed file in 0.049 seconds -[default0]: total number of samples: 4451 -[default0]: total number of epochs: 1 -[default0]:> building indices for blendable datasets ... -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default0]: > sample ratios: -[default0]: dataset 0, input: 0.0330676, achieved: 0.0330676 -[default0]: dataset 1, input: 0.0112421, achieved: 0.0112421 -[default0]: dataset 2, input: 0.130272, achieved: 0.130272 -[default0]: dataset 3, input: 0.221712, achieved: 0.221712 -[default0]: dataset 4, input: 0.106678, achieved: 0.106678 -[default0]: dataset 5, input: 0.00155951, achieved: 0.00155955 -[default0]: dataset 6, input: 0.13054, achieved: 0.13054 -[default0]: dataset 7, input: 0.010918, achieved: 0.0109181 -[default0]: dataset 8, input: 0.000110214, achieved: 0.000110257 -[default0]: dataset 9, input: 0.00549238, achieved: 0.00549235 -[default0]: dataset 10, input: 0.000402122, achieved: 0.000402094 -[default0]: dataset 11, input: 0.00747007, achieved: 0.00747007 -[default0]: dataset 12, input: 0.000619047, achieved: 0.000619024 -[default0]: dataset 13, input: 0.00103353, achieved: 0.0010336 -[default0]: dataset 14, input: 0.000501201, achieved: 0.000501226 -[default0]: dataset 15, input: 0.000667277, achieved: 0.000667231 -[default0]: dataset 16, input: 0.000359281, achieved: 0.000359326 -[default0]: dataset 17, input: 0.000508443, achieved: 0.000508519 -[default0]: dataset 18, input: 0.00211373, achieved: 0.0021138 -[default0]: dataset 19, input: 0.000912995, achieved: 0.000912961 -[default0]: dataset 20, input: 0.00124543, achieved: 0.00124546 -[default0]: dataset 21, input: 0.000315887, achieved: 0.00031594 -[default0]: dataset 22, input: 0.0813721, achieved: 0.0813721 -[default0]: dataset 23, input: 0.0552939, achieved: 0.0552939 -[default0]: dataset 24, input: 0.0495415, achieved: 0.0495414 -[default0]: dataset 25, input: 0.0246164, achieved: 0.0246163 -[default0]: dataset 26, input: 0.120917, achieved: 0.120917 -[default0]: dataset 27, input: 0.000517703, achieved: 0.000517666 -[default0]:> elapsed time for building blendable dataset indices: 0.32 (sec) -[default0]:> finished creating T0 datasets ... -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default1]:GOTCONSUMEDSAMPLES 10240 0 -[default2]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default7]:GOTCONSUMEDSAMPLES 10240 0 -[default5]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default4]:GOTCONSUMEDSAMPLES 10240 0 -[default6]:GOTCONSUMEDSAMPLES 10240 0 -[default3]:GOTCONSUMEDSAMPLES 10240 0 -[default0]:[000-046] 177.5835B / 177.5835B -[default0]:[000-022] 177.5835B / 177.5835B -[default4]:[000-001] 177.5835B / 177.5835B -[default0]:[000-028] 177.5835B / 177.5835B -[default0]:[000-010] 177.5835B / 177.5835B -[default0]:[000-036] 177.5835B / 177.5835B -[default4]:[000-037] 177.5835B / 177.5835B -[default0]:[000-042] 177.5835B / 177.5835B -[default0]:[000-068] 177.5835B / 177.5835B -[default4]:[000-069] 177.5835B / 177.5835B -[default4]:[000-035] 177.5835B / 177.5835B -[default0]:[000-056] 177.5835B / 177.5835B -[default4]:[000-015] 177.5835B / 177.5835B -[default0]:[000-030] 177.5835B / 177.5835B -[default4]:[000-049] 177.5835B / 177.5835B -[default0]:[000-018] 177.5835B / 177.5835B -[default4]:[000-041] 177.5835B / 177.5835B -[default4]:[000-057] 177.5835B / 177.5835B -[default0]:[000-032] 177.5835B / 177.5835B -[default4]:[000-033] 177.5835B / 177.5835B -[default4]:[000-031] 177.5835B / 177.5835B -[default4]:[000-011] 177.5835B / 177.5835B -[default0]:[000-016] 177.5835B / 177.5835B -[default4]:[000-017] 177.5835B / 177.5835B -[default0]:[000-006] 177.5835B / 177.5835B -[default4]:[000-003] 177.5835B / 177.5835B -[default0]:[000-066] 177.5835B / 177.5835B -[default0]:[000-038] 177.5835B / 177.5835B -[default0]:[000-044] 177.5835B / 177.5835B -[default0]:[000-008] 177.5835B / 177.5835B -[default0]:[000-052] 177.5835B / 177.5835B -[default4]:[000-071] 258.9563B / 0.0000B -[default4]:[000-065] 177.5835B / 177.5835B -[default0]:[000-026] 177.5835B / 177.5835B -[default4]:[000-005] 177.5835B / 177.5835B -[default0]:[000-004] 177.5835B / 177.5835B -[default4]:[000-013] 177.5835B / 177.5835B -[default0]:[000-012] 177.5835B / 177.5835B -[default0]:[000-054] 177.5835B / 177.5835B -[default4]:[000-051] 177.5835B / 177.5835B -[default4]:[000-067] 177.5835B / 177.5835B -[default0]:[000-040] 177.5835B / 177.5835B -[default4]:[000-029] 177.5835B / 177.5835B -[default0]:[000-060] 177.5835B / 177.5835B -[default4]:[000-045] 177.5835B / 177.5835B -[default4]:[000-019] 177.5835B / 177.5835B -[default4]:[000-063] 177.5835B / 177.5835B -[default4]:[000-007] 177.5835B / 177.5835B -[default4]:[000-021] 177.5835B / 177.5835B -[default4]:[000-059] 177.5835B / 177.5835B -[default0]:[000-058] 177.5835B / 177.5835B -[default0]:[000-070] 177.5855B / 177.5855B -[default0]:[000-024] 177.5835B / 177.5835B -[default0]:[000-048] 177.5835B / 177.5835B -[default0]:[000-034] 177.5835B / 177.5835B -[default4]:[000-039] 177.5835B / 177.5835B -[default0]:[000-062] 177.5835B / 177.5835B -[default0]:[000-050] 177.5835B / 177.5835B -[default0]:[000-002] 177.5835B / 177.5835B -[default0]:[000-014] 177.5835B / 177.5835B -[default4]:[000-055] 177.5835B / 177.5835B -[default0]:[after dataloaders are built] datetime: 2022-09-07 22:15:29 -[default0]:done with setup ... -[default0]:training ... -[default0]:Number of parameters: [tensor rank - pipeline rank] w/ and w/o embeddings: -[default0]:[000-000] 258.9584B / 0.0021B -[default0]:[before the start of training step] datetime: 2022-09-07 22:15:29 -[default4]:[000-027] 177.5835B / 177.5835B -[default0]:[000-064] 177.5835B / 177.5835B -[default0]:[000-020] 177.5835B / 177.5835B -[default4]:[000-043] 177.5835B / 177.5835B -[default4]:[000-025] 177.5835B / 177.5835B -[default4]:[000-047] 177.5835B / 177.5835B -[default4]:[000-061] 177.5835B / 177.5835B -[default4]:[000-009] 177.5835B / 177.5835B -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:[000-053] 177.5835B / 177.5835B -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default7]:time (ms) | model-and-optimizer-setup: 32123.58 | train/valid/test-data-iterators-setup: 29426.99 -[default4]:[000-023] 177.5835B / 177.5835B -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default3]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default3]: return self._grad -[default0]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default0]: return self._grad -[default2]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default2]: return self._grad -[default1]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default1]: return self._grad -[default6]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default6]: return self._grad -[default5]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default5]: return self._grad -[default7]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default7]: return self._grad -[default4]:/gpfswork/rech/six/commun/conda/tr13f-6B3-ml-t0/lib/python3.8/site-packages/torch/_tensor.py:1083: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352465323/work/build/aten/src/ATen/core/TensorBody.h:477.) -[default4]: return self._grad -[default0]:[Rank 272] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 29998.39501953125 | reserved: 35286.0 | max reserved: 35286.0 -[default0]:[Rank 224] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32110.39501953125 | reserved: 37974.0 | max reserved: 37974.0 -[default0]:[Rank 152] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35278.39501953125 | reserved: 40662.0 | max reserved: 40662.0 -[default4]:[Rank 132] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 36158.39501953125 | reserved: 41558.0 | max reserved: 41558.0 -[default0]:[Rank 128] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 36334.39501953125 | reserved: 42734.0 | max reserved: 42734.0 -[default4]:[Rank 12] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 41438.39501953125 | reserved: 46934.0 | max reserved: 46934.0 -[default0]:[Rank 120] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 36686.39501953125 | reserved: 42454.0 | max reserved: 42454.0 -[default4]:[Rank 44] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40030.39501953125 | reserved: 46318.0 | max reserved: 46318.0 -[default4]:[Rank 164] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34750.39501953125 | reserved: 40942.0 | max reserved: 40942.0 -[default0]:[Rank 176] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34222.39501953125 | reserved: 39766.0 | max reserved: 39766.0 -[default0]:[Rank 64] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 39150.39501953125 | reserved: 45422.0 | max reserved: 45422.0 -[default4]:[Rank 140] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35806.39501953125 | reserved: 41558.0 | max reserved: 41558.0 -[default0]:[Rank 72] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38798.39501953125 | reserved: 44246.0 | max reserved: 44246.0 -[default4]:[Rank 124] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 36510.39501953125 | reserved: 42454.0 | max reserved: 42454.0 -[default4]:[Rank 156] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35102.39501953125 | reserved: 40662.0 | max reserved: 40662.0 -[default0]:[Rank 216] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32462.39501953125 | reserved: 37974.0 | max reserved: 37974.0 -[default4]:[Rank 260] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 30526.39501953125 | reserved: 36182.0 | max reserved: 36182.0 -[default4]:[Rank 228] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31934.39501953125 | reserved: 38254.0 | max reserved: 38254.0 -[default4]:[Rank 196] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 33342.39501953125 | reserved: 38870.0 | max reserved: 38870.0 -[default0]:[Rank 24] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40910.39501953125 | reserved: 47214.0 | max reserved: 47214.0 -[default4]:[Rank 180] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34046.39501953125 | reserved: 39766.0 | max reserved: 39766.0 -[default0]:[Rank 280] (after 6 iterations) memory (MB) | allocated: 25990.69677734375 | max allocated: 29702.71142578125 | reserved: 35286.0 | max reserved: 35286.0 -[default4]:[Rank 284] (after 6 iterations) memory (MB) | allocated: 41930.33251953125 | max allocated: 55650.33203125 | reserved: 73748.0 | max reserved: 73748.0 -[default4]:[Rank 236] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31582.39501953125 | reserved: 37078.0 | max reserved: 37078.0 -[default0]:[Rank 160] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34926.39501953125 | reserved: 40662.0 | max reserved: 40662.0 -[default0]:[Rank 232] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31758.39501953125 | reserved: 38254.0 | max reserved: 38254.0 -[default0]:[Rank 264] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 30350.39501953125 | reserved: 36182.0 | max reserved: 36182.0 -[default0]:[Rank 32] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40558.39501953125 | reserved: 46038.0 | max reserved: 46038.0 -[default4]:[Rank 252] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 30878.39501953125 | reserved: 37358.0 | max reserved: 37358.0 -[default0]:[Rank 48] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 39854.39501953125 | reserved: 45142.0 | max reserved: 45142.0 -[default0]:[Rank 208] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32814.39501953125 | reserved: 39150.0 | max reserved: 39150.0 -[default4]:[Rank 52] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 39678.39501953125 | reserved: 45142.0 | max reserved: 45142.0 -[default0]:[Rank 56] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 39502.39501953125 | reserved: 45142.0 | max reserved: 45142.0 -[default0]:[Rank 104] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37390.39501953125 | reserved: 43350.0 | max reserved: 43350.0 -[default4]:[Rank 268] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 30174.39501953125 | reserved: 35286.0 | max reserved: 35286.0 -[default4]:[Rank 76] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38622.39501953125 | reserved: 44246.0 | max reserved: 44246.0 -[default0]:[Rank 240] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31406.39501953125 | reserved: 37078.0 | max reserved: 37078.0 -[default4]:[Rank 84] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38270.39501953125 | reserved: 44526.0 | max reserved: 44526.0 -[default4]:[Rank 68] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38974.39501953125 | reserved: 44246.0 | max reserved: 44246.0 -[default0]:[Rank 96] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37742.39501953125 | reserved: 43350.0 | max reserved: 43350.0 -[default0]:[Rank 192] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 33518.39501953125 | reserved: 40046.0 | max reserved: 40046.0 -[default4]:[Rank 20] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 41086.39501953125 | reserved: 46934.0 | max reserved: 46934.0 -[default0]:[Rank 248] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31054.39501953125 | reserved: 37358.0 | max reserved: 37358.0 -[default4]:[Rank 204] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32990.39501953125 | reserved: 38870.0 | max reserved: 38870.0 -[default4]:[Rank 116] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 36862.39501953125 | reserved: 42454.0 | max reserved: 42454.0 -[default0]:[Rank 0] (after 6 iterations) memory (MB) | allocated: 38080.58544921875 | max allocated: 62086.80322265625 | reserved: 76022.0 | max reserved: 76022.0 -[default0]:[Rank 16] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 41262.39501953125 | reserved: 46934.0 | max reserved: 46934.0 -[default4]:[Rank 220] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32286.39501953125 | reserved: 37974.0 | max reserved: 37974.0 -[default0]:[Rank 8] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 41614.39501953125 | reserved: 46934.0 | max reserved: 46934.0 -[default0]:[Rank 136] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35982.39501953125 | reserved: 41558.0 | max reserved: 41558.0 -[default4]:[Rank 108] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37214.39501953125 | reserved: 43630.0 | max reserved: 43630.0 -[default4]:[Rank 28] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40734.39501953125 | reserved: 46038.0 | max reserved: 46038.0 -[default0]:[Rank 200] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 33166.39501953125 | reserved: 38870.0 | max reserved: 38870.0 -[default0]:[Rank 80] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38446.39501953125 | reserved: 44246.0 | max reserved: 44246.0 -[default4]:[Rank 172] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34398.39501953125 | reserved: 39766.0 | max reserved: 39766.0 -[default0]:[Rank 256] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 30702.39501953125 | reserved: 37358.0 | max reserved: 37358.0 -[default4]:[Rank 244] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 31230.39501953125 | reserved: 37078.0 | max reserved: 37078.0 -[default4]:[Rank 100] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37566.39501953125 | reserved: 43350.0 | max reserved: 43350.0 -[default4]:[Rank 188] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 33694.39501953125 | reserved: 40046.0 | max reserved: 40046.0 -[default4]:[Rank 36] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40382.39501953125 | reserved: 46038.0 | max reserved: 46038.0 -[default4]:[Rank 92] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37918.39501953125 | reserved: 43350.0 | max reserved: 43350.0 -[default4]:[Rank 212] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 32638.39501953125 | reserved: 37974.0 | max reserved: 37974.0 -[default0]:[Rank 184] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 33870.39501953125 | reserved: 39766.0 | max reserved: 39766.0 -[default7]: iteration 6/ 3100 | consumed samples: 12288 | consumed tokens: 25165824 | elapsed time per iteration (s): 207.23 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.847781E+00 | grad norm: 2.680 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 9.883 | TFLOPs: 100.89 | -[default0]:[Rank 88] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 38094.39501953125 | reserved: 44526.0 | max reserved: 44526.0 -[default4]:[Rank 4] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 41790.39501953125 | reserved: 48110.0 | max reserved: 48110.0 -[default0]:[Rank 112] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 37038.39501953125 | reserved: 42454.0 | max reserved: 42454.0 -[default0]:[Rank 144] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35630.39501953125 | reserved: 41558.0 | max reserved: 41558.0 -[default4]:[Rank 148] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 35454.39501953125 | reserved: 41838.0 | max reserved: 41838.0 -[default0]:[Rank 40] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 40206.39501953125 | reserved: 46038.0 | max reserved: 46038.0 -[default4]:[Rank 276] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 29822.39501953125 | reserved: 35286.0 | max reserved: 35286.0 -[default4]:[Rank 60] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 39326.39501953125 | reserved: 45142.0 | max reserved: 45142.0 -[default0]:[Rank 168] (after 6 iterations) memory (MB) | allocated: 25990.39599609375 | max allocated: 34574.39501953125 | reserved: 40942.0 | max reserved: 40942.0 -[default7]: iteration 7/ 3100 | consumed samples: 14336 | consumed tokens: 29360128 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.830238E+00 | grad norm: 4.302 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 8/ 3100 | consumed samples: 16384 | consumed tokens: 33554432 | elapsed time per iteration (s): 143.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.777899E+00 | grad norm: 2.318 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.261 | TFLOPs: 145.58 | -[default7]: iteration 9/ 3100 | consumed samples: 18432 | consumed tokens: 37748736 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.745038E+00 | grad norm: 2.786 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 10/ 3100 | consumed samples: 20480 | consumed tokens: 41943040 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.718095E+00 | grad norm: 2.323 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 11/ 3100 | consumed samples: 22528 | consumed tokens: 46137344 | elapsed time per iteration (s): 141.14 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.686703E+00 | grad norm: 1.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.510 | TFLOPs: 148.12 | -[default7]: iteration 12/ 3100 | consumed samples: 24576 | consumed tokens: 50331648 | elapsed time per iteration (s): 140.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.660032E+00 | grad norm: 1.677 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.569 | TFLOPs: 148.73 | -[default7]: iteration 13/ 3100 | consumed samples: 26624 | consumed tokens: 54525952 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.645087E+00 | grad norm: 1.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.47 | -[default7]: iteration 14/ 3100 | consumed samples: 28672 | consumed tokens: 58720256 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.628401E+00 | grad norm: 1.012 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 15/ 3100 | consumed samples: 30720 | consumed tokens: 62914560 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.617762E+00 | grad norm: 2.262 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 16/ 3100 | consumed samples: 32768 | consumed tokens: 67108864 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.624299E+00 | grad norm: 1.664 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 17/ 3100 | consumed samples: 34816 | consumed tokens: 71303168 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.602368E+00 | grad norm: 2.993 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 18/ 3100 | consumed samples: 36864 | consumed tokens: 75497472 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.597052E+00 | grad norm: 1.830 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 19/ 3100 | consumed samples: 38912 | consumed tokens: 79691776 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.593526E+00 | grad norm: 0.892 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 20/ 3100 | consumed samples: 40960 | consumed tokens: 83886080 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.575442E+00 | grad norm: 1.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 21/ 3100 | consumed samples: 43008 | consumed tokens: 88080384 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.575056E+00 | grad norm: 1.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 22/ 3100 | consumed samples: 45056 | consumed tokens: 92274688 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.561691E+00 | grad norm: 1.062 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 23/ 3100 | consumed samples: 47104 | consumed tokens: 96468992 | elapsed time per iteration (s): 140.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.565794E+00 | grad norm: 1.053 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.529 | TFLOPs: 148.32 | -[default7]: iteration 24/ 3100 | consumed samples: 49152 | consumed tokens: 100663296 | elapsed time per iteration (s): 140.05 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.551106E+00 | grad norm: 0.932 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.624 | TFLOPs: 149.29 | -[default7]: iteration 25/ 3100 | consumed samples: 51200 | consumed tokens: 104857600 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.535627E+00 | grad norm: 1.002 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 26/ 3100 | consumed samples: 53248 | consumed tokens: 109051904 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.532577E+00 | grad norm: 0.897 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 27/ 3100 | consumed samples: 55296 | consumed tokens: 113246208 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.537946E+00 | grad norm: 0.972 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 28/ 3100 | consumed samples: 57344 | consumed tokens: 117440512 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.523635E+00 | grad norm: 0.847 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 29/ 3100 | consumed samples: 59392 | consumed tokens: 121634816 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.518522E+00 | grad norm: 0.890 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 30/ 3100 | consumed samples: 61440 | consumed tokens: 125829120 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.514112E+00 | grad norm: 0.667 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 31/ 3100 | consumed samples: 63488 | consumed tokens: 130023424 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.519079E+00 | grad norm: 1.003 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 32/ 3100 | consumed samples: 65536 | consumed tokens: 134217728 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.498534E+00 | grad norm: 0.829 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 33/ 3100 | consumed samples: 67584 | consumed tokens: 138412032 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.498235E+00 | grad norm: 0.858 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 34/ 3100 | consumed samples: 69632 | consumed tokens: 142606336 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.493934E+00 | grad norm: 0.589 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 35/ 3100 | consumed samples: 71680 | consumed tokens: 146800640 | elapsed time per iteration (s): 140.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.481980E+00 | grad norm: 0.862 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.566 | TFLOPs: 148.70 | -[default7]: iteration 36/ 3100 | consumed samples: 73728 | consumed tokens: 150994944 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.485864E+00 | grad norm: 0.978 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 37/ 3100 | consumed samples: 75776 | consumed tokens: 155189248 | elapsed time per iteration (s): 140.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.491511E+00 | grad norm: 0.774 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.574 | TFLOPs: 148.78 | -[default7]: iteration 38/ 3100 | consumed samples: 77824 | consumed tokens: 159383552 | elapsed time per iteration (s): 140.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.495243E+00 | grad norm: 0.634 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.548 | TFLOPs: 148.51 | -[default7]: iteration 39/ 3100 | consumed samples: 79872 | consumed tokens: 163577856 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.471316E+00 | grad norm: 0.632 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 40/ 3100 | consumed samples: 81920 | consumed tokens: 167772160 | elapsed time per iteration (s): 141.19 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.477593E+00 | grad norm: 0.549 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.506 | TFLOPs: 148.08 | -[default7]: iteration 41/ 3100 | consumed samples: 83968 | consumed tokens: 171966464 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.472040E+00 | grad norm: 0.701 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.27 | -[default7]: iteration 42/ 3100 | consumed samples: 86016 | consumed tokens: 176160768 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.475639E+00 | grad norm: 0.619 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 43/ 3100 | consumed samples: 88064 | consumed tokens: 180355072 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.460621E+00 | grad norm: 0.623 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 44/ 3100 | consumed samples: 90112 | consumed tokens: 184549376 | elapsed time per iteration (s): 140.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.473720E+00 | grad norm: 0.616 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.562 | TFLOPs: 148.66 | -[default7]: iteration 45/ 3100 | consumed samples: 92160 | consumed tokens: 188743680 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.456498E+00 | grad norm: 0.772 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 46/ 3100 | consumed samples: 94208 | consumed tokens: 192937984 | elapsed time per iteration (s): 140.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.457895E+00 | grad norm: 0.628 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.550 | TFLOPs: 148.53 | -[default7]: iteration 47/ 3100 | consumed samples: 96256 | consumed tokens: 197132288 | elapsed time per iteration (s): 141.24 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.448618E+00 | grad norm: 0.673 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.500 | TFLOPs: 148.02 | -[default7]: iteration 48/ 3100 | consumed samples: 98304 | consumed tokens: 201326592 | elapsed time per iteration (s): 150.18 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.453855E+00 | grad norm: 0.695 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 13.637 | TFLOPs: 139.22 | -[default7]: iteration 49/ 3100 | consumed samples: 100352 | consumed tokens: 205520896 | elapsed time per iteration (s): 140.30 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.446694E+00 | grad norm: 0.533 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.597 | TFLOPs: 149.01 | -[default7]: iteration 50/ 3100 | consumed samples: 102400 | consumed tokens: 209715200 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.446030E+00 | grad norm: 0.617 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.51 | -[default7]: iteration 51/ 3100 | consumed samples: 104448 | consumed tokens: 213909504 | elapsed time per iteration (s): 141.13 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.447621E+00 | grad norm: 0.633 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.511 | TFLOPs: 148.14 | -[default7]: iteration 52/ 3100 | consumed samples: 106496 | consumed tokens: 218103808 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.458813E+00 | grad norm: 0.635 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 53/ 3100 | consumed samples: 108544 | consumed tokens: 222298112 | elapsed time per iteration (s): 141.27 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.446339E+00 | grad norm: 0.591 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.497 | TFLOPs: 148.00 | -[default7]: iteration 54/ 3100 | consumed samples: 110592 | consumed tokens: 226492416 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.438542E+00 | grad norm: 0.649 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 55/ 3100 | consumed samples: 112640 | consumed tokens: 230686720 | elapsed time per iteration (s): 140.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.439770E+00 | grad norm: 0.613 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.583 | TFLOPs: 148.87 | -[default7]: iteration 56/ 3100 | consumed samples: 114688 | consumed tokens: 234881024 | elapsed time per iteration (s): 153.30 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.438952E+00 | grad norm: 0.609 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 13.359 | TFLOPs: 136.38 | -[default7]: iteration 57/ 3100 | consumed samples: 116736 | consumed tokens: 239075328 | elapsed time per iteration (s): 152.21 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.428383E+00 | grad norm: 0.556 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 13.455 | TFLOPs: 137.35 | -[default7]: iteration 58/ 3100 | consumed samples: 118784 | consumed tokens: 243269632 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.440760E+00 | grad norm: 0.613 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 59/ 3100 | consumed samples: 120832 | consumed tokens: 247463936 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.434417E+00 | grad norm: 0.570 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 60/ 3100 | consumed samples: 122880 | consumed tokens: 251658240 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.425971E+00 | grad norm: 0.578 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 61/ 3100 | consumed samples: 124928 | consumed tokens: 255852544 | elapsed time per iteration (s): 140.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.413134E+00 | grad norm: 0.672 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.552 | TFLOPs: 148.56 | -[default7]: iteration 62/ 3100 | consumed samples: 126976 | consumed tokens: 260046848 | elapsed time per iteration (s): 140.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.430684E+00 | grad norm: 0.708 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.595 | TFLOPs: 148.99 | -[default7]: iteration 63/ 3100 | consumed samples: 129024 | consumed tokens: 264241152 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.411595E+00 | grad norm: 0.637 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 64/ 3100 | consumed samples: 131072 | consumed tokens: 268435456 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.422456E+00 | grad norm: 0.874 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 65/ 3100 | consumed samples: 133120 | consumed tokens: 272629760 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.414503E+00 | grad norm: 0.553 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 66/ 3100 | consumed samples: 135168 | consumed tokens: 276824064 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.407192E+00 | grad norm: 0.882 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 67/ 3100 | consumed samples: 137216 | consumed tokens: 281018368 | elapsed time per iteration (s): 141.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.414502E+00 | grad norm: 0.633 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.484 | TFLOPs: 147.86 | -[default7]: iteration 68/ 3100 | consumed samples: 139264 | consumed tokens: 285212672 | elapsed time per iteration (s): 142.35 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.408151E+00 | grad norm: 0.924 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.387 | TFLOPs: 146.87 | -[default7]: iteration 69/ 3100 | consumed samples: 141312 | consumed tokens: 289406976 | elapsed time per iteration (s): 140.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.395499E+00 | grad norm: 5.256 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.556 | TFLOPs: 148.60 | -[default7]: iteration 70/ 3100 | consumed samples: 143360 | consumed tokens: 293601280 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.406005E+00 | grad norm: 0.804 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 71/ 3100 | consumed samples: 145408 | consumed tokens: 297795584 | elapsed time per iteration (s): 141.15 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.412574E+00 | grad norm: 0.729 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.509 | TFLOPs: 148.12 | -[default7]: iteration 72/ 3100 | consumed samples: 147456 | consumed tokens: 301989888 | elapsed time per iteration (s): 140.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.406932E+00 | grad norm: 1.168 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.580 | TFLOPs: 148.84 | -[default7]: iteration 73/ 3100 | consumed samples: 149504 | consumed tokens: 306184192 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.385754E+00 | grad norm: 0.637 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 74/ 3100 | consumed samples: 151552 | consumed tokens: 310378496 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.392721E+00 | grad norm: 0.782 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 75/ 3100 | consumed samples: 153600 | consumed tokens: 314572800 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.418246E+00 | grad norm: 0.563 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 76/ 3100 | consumed samples: 155648 | consumed tokens: 318767104 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.408500E+00 | grad norm: 0.572 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 77/ 3100 | consumed samples: 157696 | consumed tokens: 322961408 | elapsed time per iteration (s): 140.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.382388E+00 | grad norm: 1.307 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.587 | TFLOPs: 148.91 | -[default7]: iteration 78/ 3100 | consumed samples: 159744 | consumed tokens: 327155712 | elapsed time per iteration (s): 140.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.394898E+00 | grad norm: 0.721 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.579 | TFLOPs: 148.83 | -[default7]: iteration 79/ 3100 | consumed samples: 161792 | consumed tokens: 331350016 | elapsed time per iteration (s): 141.28 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.407915E+00 | grad norm: 0.555 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.496 | TFLOPs: 147.98 | -[default7]: iteration 80/ 3100 | consumed samples: 163840 | consumed tokens: 335544320 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.391231E+00 | grad norm: 0.655 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.34 | -[default7]: iteration 81/ 3100 | consumed samples: 165888 | consumed tokens: 339738624 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.388843E+00 | grad norm: 0.545 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.427 | TFLOPs: 147.27 | -[default7]: iteration 82/ 3100 | consumed samples: 167936 | consumed tokens: 343932928 | elapsed time per iteration (s): 139.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.385740E+00 | grad norm: 0.571 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.639 | TFLOPs: 149.44 | -[default7]: iteration 83/ 3100 | consumed samples: 169984 | consumed tokens: 348127232 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.385725E+00 | grad norm: 0.562 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 84/ 3100 | consumed samples: 172032 | consumed tokens: 352321536 | elapsed time per iteration (s): 140.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.400836E+00 | grad norm: 0.548 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.554 | TFLOPs: 148.57 | -[default7]: iteration 85/ 3100 | consumed samples: 174080 | consumed tokens: 356515840 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.368668E+00 | grad norm: 0.543 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 86/ 3100 | consumed samples: 176128 | consumed tokens: 360710144 | elapsed time per iteration (s): 145.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.385807E+00 | grad norm: 0.676 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.068 | TFLOPs: 143.61 | -[default7]: iteration 87/ 3100 | consumed samples: 178176 | consumed tokens: 364904448 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.383048E+00 | grad norm: 0.543 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 88/ 3100 | consumed samples: 180224 | consumed tokens: 369098752 | elapsed time per iteration (s): 142.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.374536E+00 | grad norm: 0.606 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.332 | TFLOPs: 146.31 | -[default7]: iteration 89/ 3100 | consumed samples: 182272 | consumed tokens: 373293056 | elapsed time per iteration (s): 140.35 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.365944E+00 | grad norm: 0.577 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.592 | TFLOPs: 148.96 | -[default7]: iteration 90/ 3100 | consumed samples: 184320 | consumed tokens: 377487360 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.369945E+00 | grad norm: 0.545 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 91/ 3100 | consumed samples: 186368 | consumed tokens: 381681664 | elapsed time per iteration (s): 140.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.368766E+00 | grad norm: 0.559 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.564 | TFLOPs: 148.67 | -[default7]: iteration 92/ 3100 | consumed samples: 188416 | consumed tokens: 385875968 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.374578E+00 | grad norm: 0.552 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 93/ 3100 | consumed samples: 190464 | consumed tokens: 390070272 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.352346E+00 | grad norm: 0.542 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 94/ 3100 | consumed samples: 192512 | consumed tokens: 394264576 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.373470E+00 | grad norm: 1.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 95/ 3100 | consumed samples: 194560 | consumed tokens: 398458880 | elapsed time per iteration (s): 141.06 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.351131E+00 | grad norm: 0.602 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.519 | TFLOPs: 148.22 | -[default7]: iteration 96/ 3100 | consumed samples: 196608 | consumed tokens: 402653184 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.366678E+00 | grad norm: 0.559 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.43 | -[default7]: iteration 97/ 3100 | consumed samples: 198656 | consumed tokens: 406847488 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.372962E+00 | grad norm: 0.589 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 98/ 3100 | consumed samples: 200704 | consumed tokens: 411041792 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.354619E+00 | grad norm: 0.504 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 99/ 3100 | consumed samples: 202752 | consumed tokens: 415236096 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.361352E+00 | grad norm: 0.572 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 100/ 3100 | consumed samples: 204800 | consumed tokens: 419430400 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.367922E+00 | grad norm: 0.507 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 101/ 3100 | consumed samples: 206848 | consumed tokens: 423624704 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.358018E+00 | grad norm: 0.564 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 102/ 3100 | consumed samples: 208896 | consumed tokens: 427819008 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.357660E+00 | grad norm: 0.506 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 103/ 3100 | consumed samples: 210944 | consumed tokens: 432013312 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.348413E+00 | grad norm: 0.573 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 104/ 3100 | consumed samples: 212992 | consumed tokens: 436207616 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.337660E+00 | grad norm: 0.526 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 105/ 3100 | consumed samples: 215040 | consumed tokens: 440401920 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.350495E+00 | grad norm: 0.619 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 106/ 3100 | consumed samples: 217088 | consumed tokens: 444596224 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.363093E+00 | grad norm: 0.548 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 107/ 3100 | consumed samples: 219136 | consumed tokens: 448790528 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.349876E+00 | grad norm: 0.685 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 108/ 3100 | consumed samples: 221184 | consumed tokens: 452984832 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.344851E+00 | grad norm: 0.601 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 109/ 3100 | consumed samples: 223232 | consumed tokens: 457179136 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.357136E+00 | grad norm: 0.570 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 110/ 3100 | consumed samples: 225280 | consumed tokens: 461373440 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.332577E+00 | grad norm: 0.602 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 111/ 3100 | consumed samples: 227328 | consumed tokens: 465567744 | elapsed time per iteration (s): 146.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.348041E+00 | grad norm: 1.740 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.003 | TFLOPs: 142.95 | -[default7]: iteration 112/ 3100 | consumed samples: 229376 | consumed tokens: 469762048 | elapsed time per iteration (s): 142.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.330516E+00 | grad norm: 0.827 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.328 | TFLOPs: 146.27 | -[default7]: iteration 113/ 3100 | consumed samples: 231424 | consumed tokens: 473956352 | elapsed time per iteration (s): 142.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.332081E+00 | grad norm: 0.575 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.372 | TFLOPs: 146.72 | -[default7]: iteration 114/ 3100 | consumed samples: 233472 | consumed tokens: 478150656 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.328644E+00 | grad norm: 0.885 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 115/ 3100 | consumed samples: 235520 | consumed tokens: 482344960 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.339532E+00 | grad norm: 3.714 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 116/ 3100 | consumed samples: 237568 | consumed tokens: 486539264 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.342600E+00 | grad norm: 1.184 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 117/ 3100 | consumed samples: 239616 | consumed tokens: 490733568 | elapsed time per iteration (s): 141.10 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.330705E+00 | grad norm: 0.579 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.515 | TFLOPs: 148.17 | -[default7]: iteration 118/ 3100 | consumed samples: 241664 | consumed tokens: 494927872 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.326318E+00 | grad norm: 0.844 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 119/ 3100 | consumed samples: 243712 | consumed tokens: 499122176 | elapsed time per iteration (s): 142.35 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.326915E+00 | grad norm: 0.861 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.387 | TFLOPs: 146.87 | -[default7]: iteration 120/ 3100 | consumed samples: 245760 | consumed tokens: 503316480 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.311904E+00 | grad norm: 0.553 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 121/ 3100 | consumed samples: 247808 | consumed tokens: 507510784 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.328532E+00 | grad norm: 1.183 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 122/ 3100 | consumed samples: 249856 | consumed tokens: 511705088 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.322095E+00 | grad norm: 0.856 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 123/ 3100 | consumed samples: 251904 | consumed tokens: 515899392 | elapsed time per iteration (s): 143.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.326992E+00 | grad norm: 0.637 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.244 | TFLOPs: 145.41 | -[default7]: iteration 124/ 3100 | consumed samples: 253952 | consumed tokens: 520093696 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.316971E+00 | grad norm: 0.809 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 125/ 3100 | consumed samples: 256000 | consumed tokens: 524288000 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.303779E+00 | grad norm: 0.534 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 126/ 3100 | consumed samples: 258048 | consumed tokens: 528482304 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.327411E+00 | grad norm: 0.831 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 127/ 3100 | consumed samples: 260096 | consumed tokens: 532676608 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.311250E+00 | grad norm: 0.719 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 128/ 3100 | consumed samples: 262144 | consumed tokens: 536870912 | elapsed time per iteration (s): 140.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.309731E+00 | grad norm: 0.543 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.532 | TFLOPs: 148.35 | -[default7]: iteration 129/ 3100 | consumed samples: 264192 | consumed tokens: 541065216 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.313187E+00 | grad norm: 0.659 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 130/ 3100 | consumed samples: 266240 | consumed tokens: 545259520 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.293942E+00 | grad norm: 0.647 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 131/ 3100 | consumed samples: 268288 | consumed tokens: 549453824 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.308722E+00 | grad norm: 0.466 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 132/ 3100 | consumed samples: 270336 | consumed tokens: 553648128 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.320434E+00 | grad norm: 0.625 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 133/ 3100 | consumed samples: 272384 | consumed tokens: 557842432 | elapsed time per iteration (s): 142.05 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.309332E+00 | grad norm: 0.541 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.418 | TFLOPs: 147.18 | -[default7]: iteration 134/ 3100 | consumed samples: 274432 | consumed tokens: 562036736 | elapsed time per iteration (s): 140.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.314828E+00 | grad norm: 0.519 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.578 | TFLOPs: 148.82 | -[default7]: iteration 135/ 3100 | consumed samples: 276480 | consumed tokens: 566231040 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.302442E+00 | grad norm: 0.640 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 136/ 3100 | consumed samples: 278528 | consumed tokens: 570425344 | elapsed time per iteration (s): 143.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.293788E+00 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.321 | TFLOPs: 146.20 | -[default7]: iteration 137/ 3100 | consumed samples: 280576 | consumed tokens: 574619648 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.298861E+00 | grad norm: 0.553 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 138/ 3100 | consumed samples: 282624 | consumed tokens: 578813952 | elapsed time per iteration (s): 140.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.297856E+00 | grad norm: 0.553 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.582 | TFLOPs: 148.86 | -[default7]: iteration 139/ 3100 | consumed samples: 284672 | consumed tokens: 583008256 | elapsed time per iteration (s): 150.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.309103E+00 | grad norm: 0.529 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 13.600 | TFLOPs: 138.84 | -[default7]: iteration 140/ 3100 | consumed samples: 286720 | consumed tokens: 587202560 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.294917E+00 | grad norm: 0.459 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 141/ 3100 | consumed samples: 288768 | consumed tokens: 591396864 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.290626E+00 | grad norm: 0.496 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 142/ 3100 | consumed samples: 290816 | consumed tokens: 595591168 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.307256E+00 | grad norm: 0.624 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 143/ 3100 | consumed samples: 292864 | consumed tokens: 599785472 | elapsed time per iteration (s): 142.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.289019E+00 | grad norm: 0.532 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.375 | TFLOPs: 146.75 | -[default7]: iteration 144/ 3100 | consumed samples: 294912 | consumed tokens: 603979776 | elapsed time per iteration (s): 141.19 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.296445E+00 | grad norm: 0.544 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.505 | TFLOPs: 148.07 | -[default7]: iteration 145/ 3100 | consumed samples: 296960 | consumed tokens: 608174080 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.275099E+00 | grad norm: 0.743 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 146/ 3100 | consumed samples: 299008 | consumed tokens: 612368384 | elapsed time per iteration (s): 143.07 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.289611E+00 | grad norm: 0.541 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.315 | TFLOPs: 146.13 | -[default7]: iteration 147/ 3100 | consumed samples: 301056 | consumed tokens: 616562688 | elapsed time per iteration (s): 140.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.287970E+00 | grad norm: 0.513 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.535 | TFLOPs: 148.38 | -[default7]: iteration 148/ 3100 | consumed samples: 303104 | consumed tokens: 620756992 | elapsed time per iteration (s): 142.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.291433E+00 | grad norm: 0.512 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.343 | TFLOPs: 146.42 | -[default7]: iteration 149/ 3100 | consumed samples: 305152 | consumed tokens: 624951296 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.273355E+00 | grad norm: 0.588 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 150/ 3100 | consumed samples: 307200 | consumed tokens: 629145600 | elapsed time per iteration (s): 140.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.282526E+00 | grad norm: 0.517 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.551 | TFLOPs: 148.54 | -[default7]: iteration 151/ 3100 | consumed samples: 309248 | consumed tokens: 633339904 | elapsed time per iteration (s): 141.17 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.285502E+00 | grad norm: 0.504 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.508 | TFLOPs: 148.10 | -[default7]: iteration 152/ 3100 | consumed samples: 311296 | consumed tokens: 637534208 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.280407E+00 | grad norm: 0.532 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 153/ 3100 | consumed samples: 313344 | consumed tokens: 641728512 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.272451E+00 | grad norm: 0.488 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 154/ 3100 | consumed samples: 315392 | consumed tokens: 645922816 | elapsed time per iteration (s): 141.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.272881E+00 | grad norm: 0.536 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.484 | TFLOPs: 147.86 | -[default7]: iteration 155/ 3100 | consumed samples: 317440 | consumed tokens: 650117120 | elapsed time per iteration (s): 140.22 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.287994E+00 | grad norm: 0.631 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.606 | TFLOPs: 149.10 | -[default7]: iteration 156/ 3100 | consumed samples: 319488 | consumed tokens: 654311424 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.268383E+00 | grad norm: 0.521 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 | -[default7]: iteration 157/ 3100 | consumed samples: 321536 | consumed tokens: 658505728 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.272132E+00 | grad norm: 0.642 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 158/ 3100 | consumed samples: 323584 | consumed tokens: 662700032 | elapsed time per iteration (s): 147.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.263744E+00 | grad norm: 0.507 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 13.930 | TFLOPs: 142.20 | -[default7]: iteration 159/ 3100 | consumed samples: 325632 | consumed tokens: 666894336 | elapsed time per iteration (s): 143.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.289691E+00 | grad norm: 0.539 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.228 | TFLOPs: 145.24 | -[default7]: iteration 160/ 3100 | consumed samples: 327680 | consumed tokens: 671088640 | elapsed time per iteration (s): 141.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.266098E+00 | grad norm: 0.528 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.484 | TFLOPs: 147.86 | -[default7]: iteration 161/ 3100 | consumed samples: 329728 | consumed tokens: 675282944 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.260032E+00 | grad norm: 0.501 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 162/ 3100 | consumed samples: 331776 | consumed tokens: 679477248 | elapsed time per iteration (s): 141.35 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.277431E+00 | grad norm: 0.594 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.488 | TFLOPs: 147.90 | -[default7]: iteration 163/ 3100 | consumed samples: 333824 | consumed tokens: 683671552 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.276256E+00 | grad norm: 1.203 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 164/ 3100 | consumed samples: 335872 | consumed tokens: 687865856 | elapsed time per iteration (s): 144.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.256145E+00 | grad norm: 0.903 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.153 | TFLOPs: 144.48 | -[default7]: iteration 165/ 3100 | consumed samples: 337920 | consumed tokens: 692060160 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.280634E+00 | grad norm: 0.546 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 166/ 3100 | consumed samples: 339968 | consumed tokens: 696254464 | elapsed time per iteration (s): 156.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.271560E+00 | grad norm: 0.714 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 13.093 | TFLOPs: 133.66 | -[default7]: iteration 167/ 3100 | consumed samples: 342016 | consumed tokens: 700448768 | elapsed time per iteration (s): 143.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.264967E+00 | grad norm: 0.482 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.246 | TFLOPs: 145.43 | -[default7]: iteration 168/ 3100 | consumed samples: 344064 | consumed tokens: 704643072 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.261504E+00 | grad norm: 0.704 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 169/ 3100 | consumed samples: 346112 | consumed tokens: 708837376 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.253220E+00 | grad norm: 0.520 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 170/ 3100 | consumed samples: 348160 | consumed tokens: 713031680 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.240958E+00 | grad norm: 0.524 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 171/ 3100 | consumed samples: 350208 | consumed tokens: 717225984 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.260263E+00 | grad norm: 0.510 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 172/ 3100 | consumed samples: 352256 | consumed tokens: 721420288 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.268291E+00 | grad norm: 0.532 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 173/ 3100 | consumed samples: 354304 | consumed tokens: 725614592 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.255235E+00 | grad norm: 0.561 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 174/ 3100 | consumed samples: 356352 | consumed tokens: 729808896 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.258243E+00 | grad norm: 0.511 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 175/ 3100 | consumed samples: 358400 | consumed tokens: 734003200 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.256105E+00 | grad norm: 0.541 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.50 | -[default7]: iteration 176/ 3100 | consumed samples: 360448 | consumed tokens: 738197504 | elapsed time per iteration (s): 139.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.230968E+00 | grad norm: 0.494 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.674 | TFLOPs: 149.80 | -[default7]: iteration 177/ 3100 | consumed samples: 362496 | consumed tokens: 742391808 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.253087E+00 | grad norm: 0.578 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 178/ 3100 | consumed samples: 364544 | consumed tokens: 746586112 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.259760E+00 | grad norm: 0.526 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 179/ 3100 | consumed samples: 366592 | consumed tokens: 750780416 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.241974E+00 | grad norm: 0.538 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 180/ 3100 | consumed samples: 368640 | consumed tokens: 754974720 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.248333E+00 | grad norm: 0.504 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 181/ 3100 | consumed samples: 370688 | consumed tokens: 759169024 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.250346E+00 | grad norm: 0.540 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 182/ 3100 | consumed samples: 372736 | consumed tokens: 763363328 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.244624E+00 | grad norm: 0.879 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 183/ 3100 | consumed samples: 374784 | consumed tokens: 767557632 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.235885E+00 | grad norm: 0.505 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 184/ 3100 | consumed samples: 376832 | consumed tokens: 771751936 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.238281E+00 | grad norm: 0.487 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 185/ 3100 | consumed samples: 378880 | consumed tokens: 775946240 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.259411E+00 | grad norm: 0.632 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 186/ 3100 | consumed samples: 380928 | consumed tokens: 780140544 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.247711E+00 | grad norm: 0.555 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 187/ 3100 | consumed samples: 382976 | consumed tokens: 784334848 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.234996E+00 | grad norm: 0.726 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 188/ 3100 | consumed samples: 385024 | consumed tokens: 788529152 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.238435E+00 | grad norm: 0.530 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 189/ 3100 | consumed samples: 387072 | consumed tokens: 792723456 | elapsed time per iteration (s): 141.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.245580E+00 | grad norm: 0.665 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.485 | TFLOPs: 147.87 | -[default7]: iteration 190/ 3100 | consumed samples: 389120 | consumed tokens: 796917760 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.240947E+00 | grad norm: 0.553 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 191/ 3100 | consumed samples: 391168 | consumed tokens: 801112064 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.233675E+00 | grad norm: 0.623 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 192/ 3100 | consumed samples: 393216 | consumed tokens: 805306368 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.252792E+00 | grad norm: 0.565 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 193/ 3100 | consumed samples: 395264 | consumed tokens: 809500672 | elapsed time per iteration (s): 144.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.230776E+00 | grad norm: 0.498 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.156 | TFLOPs: 144.51 | -[default7]: iteration 194/ 3100 | consumed samples: 397312 | consumed tokens: 813694976 | elapsed time per iteration (s): 140.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.242682E+00 | grad norm: 0.523 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.629 | TFLOPs: 149.34 | -[default7]: iteration 195/ 3100 | consumed samples: 399360 | consumed tokens: 817889280 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.223703E+00 | grad norm: 0.468 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 196/ 3100 | consumed samples: 401408 | consumed tokens: 822083584 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.226636E+00 | grad norm: 0.484 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 197/ 3100 | consumed samples: 403456 | consumed tokens: 826277888 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.237658E+00 | grad norm: 0.587 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 198/ 3100 | consumed samples: 405504 | consumed tokens: 830472192 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.231080E+00 | grad norm: 0.481 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 199/ 3100 | consumed samples: 407552 | consumed tokens: 834666496 | elapsed time per iteration (s): 140.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.221713E+00 | grad norm: 0.558 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.549 | TFLOPs: 148.53 | -[default7]: iteration 200/ 3100 | consumed samples: 409600 | consumed tokens: 838860800 | elapsed time per iteration (s): 145.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.240611E+00 | grad norm: 0.526 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.068 | TFLOPs: 143.62 | -[default7]: iteration 201/ 3100 | consumed samples: 411648 | consumed tokens: 843055104 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.229626E+00 | grad norm: 0.498 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.73 | -[default7]: iteration 202/ 3100 | consumed samples: 413696 | consumed tokens: 847249408 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.238336E+00 | grad norm: 0.511 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 203/ 3100 | consumed samples: 415744 | consumed tokens: 851443712 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.236981E+00 | grad norm: 0.537 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 204/ 3100 | consumed samples: 417792 | consumed tokens: 855638016 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.222723E+00 | grad norm: 0.537 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.76 | -[default7]: iteration 205/ 3100 | consumed samples: 419840 | consumed tokens: 859832320 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.221357E+00 | grad norm: 0.522 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.54 | -[default7]: iteration 206/ 3100 | consumed samples: 421888 | consumed tokens: 864026624 | elapsed time per iteration (s): 143.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.231445E+00 | grad norm: 0.630 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.271 | TFLOPs: 145.68 | -[default7]: iteration 207/ 3100 | consumed samples: 423936 | consumed tokens: 868220928 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.221807E+00 | grad norm: 0.519 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 | -[default7]: iteration 208/ 3100 | consumed samples: 425984 | consumed tokens: 872415232 | elapsed time per iteration (s): 139.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.215088E+00 | grad norm: 0.779 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.699 | TFLOPs: 150.05 | -[default7]: iteration 209/ 3100 | consumed samples: 428032 | consumed tokens: 876609536 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.222710E+00 | grad norm: 1.353 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 210/ 3100 | consumed samples: 430080 | consumed tokens: 880803840 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.204996E+00 | grad norm: 0.506 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 211/ 3100 | consumed samples: 432128 | consumed tokens: 884998144 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.221942E+00 | grad norm: 0.518 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 212/ 3100 | consumed samples: 434176 | consumed tokens: 889192448 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.205795E+00 | grad norm: 0.549 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 213/ 3100 | consumed samples: 436224 | consumed tokens: 893386752 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.210330E+00 | grad norm: 0.521 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 214/ 3100 | consumed samples: 438272 | consumed tokens: 897581056 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.212764E+00 | grad norm: 0.603 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 215/ 3100 | consumed samples: 440320 | consumed tokens: 901775360 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.197299E+00 | grad norm: 0.681 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 216/ 3100 | consumed samples: 442368 | consumed tokens: 905969664 | elapsed time per iteration (s): 140.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.204668E+00 | grad norm: 0.659 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.588 | TFLOPs: 148.92 | -[default7]: iteration 217/ 3100 | consumed samples: 444416 | consumed tokens: 910163968 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.212447E+00 | grad norm: 0.552 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 218/ 3100 | consumed samples: 446464 | consumed tokens: 914358272 | elapsed time per iteration (s): 140.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.223636E+00 | grad norm: 0.861 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.563 | TFLOPs: 148.67 | -[default7]: iteration 219/ 3100 | consumed samples: 448512 | consumed tokens: 918552576 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.200405E+00 | grad norm: 0.546 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 220/ 3100 | consumed samples: 450560 | consumed tokens: 922746880 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.216007E+00 | grad norm: 0.696 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 221/ 3100 | consumed samples: 452608 | consumed tokens: 926941184 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.202348E+00 | grad norm: 0.576 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 222/ 3100 | consumed samples: 454656 | consumed tokens: 931135488 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.210257E+00 | grad norm: 0.447 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 223/ 3100 | consumed samples: 456704 | consumed tokens: 935329792 | elapsed time per iteration (s): 140.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.223068E+00 | grad norm: 0.613 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.603 | TFLOPs: 149.07 | -[default7]: iteration 224/ 3100 | consumed samples: 458752 | consumed tokens: 939524096 | elapsed time per iteration (s): 141.24 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.216887E+00 | grad norm: 0.578 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.500 | TFLOPs: 148.02 | -[default7]: iteration 225/ 3100 | consumed samples: 460800 | consumed tokens: 943718400 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.202992E+00 | grad norm: 0.619 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 226/ 3100 | consumed samples: 462848 | consumed tokens: 947912704 | elapsed time per iteration (s): 141.15 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.198332E+00 | grad norm: 0.506 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.510 | TFLOPs: 148.12 | -[default7]: iteration 227/ 3100 | consumed samples: 464896 | consumed tokens: 952107008 | elapsed time per iteration (s): 140.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.211546E+00 | grad norm: 0.531 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.563 | TFLOPs: 148.67 | -[default7]: iteration 228/ 3100 | consumed samples: 466944 | consumed tokens: 956301312 | elapsed time per iteration (s): 140.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.197102E+00 | grad norm: 0.520 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.560 | TFLOPs: 148.63 | -[default7]: iteration 229/ 3100 | consumed samples: 468992 | consumed tokens: 960495616 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.217169E+00 | grad norm: 0.564 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 230/ 3100 | consumed samples: 471040 | consumed tokens: 964689920 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.195603E+00 | grad norm: 0.514 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.62 | -[default7]: iteration 231/ 3100 | consumed samples: 473088 | consumed tokens: 968884224 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.216583E+00 | grad norm: 0.508 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 232/ 3100 | consumed samples: 475136 | consumed tokens: 973078528 | elapsed time per iteration (s): 140.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.191725E+00 | grad norm: 0.552 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.546 | TFLOPs: 148.49 | -[default7]: iteration 233/ 3100 | consumed samples: 477184 | consumed tokens: 977272832 | elapsed time per iteration (s): 140.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.196310E+00 | grad norm: 0.513 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.564 | TFLOPs: 148.68 | -[default7]: iteration 234/ 3100 | consumed samples: 479232 | consumed tokens: 981467136 | elapsed time per iteration (s): 143.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.192035E+00 | grad norm: 0.595 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.319 | TFLOPs: 146.17 | -[default7]: iteration 235/ 3100 | consumed samples: 481280 | consumed tokens: 985661440 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.194826E+00 | grad norm: 0.504 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 236/ 3100 | consumed samples: 483328 | consumed tokens: 989855744 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.207150E+00 | grad norm: 0.871 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.50 | -[default7]: iteration 237/ 3100 | consumed samples: 485376 | consumed tokens: 994050048 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.201920E+00 | grad norm: 0.512 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 238/ 3100 | consumed samples: 487424 | consumed tokens: 998244352 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.208049E+00 | grad norm: 0.668 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 239/ 3100 | consumed samples: 489472 | consumed tokens: 1002438656 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.187637E+00 | grad norm: 0.522 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 240/ 3100 | consumed samples: 491520 | consumed tokens: 1006632960 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.203614E+00 | grad norm: 0.500 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 241/ 3100 | consumed samples: 493568 | consumed tokens: 1010827264 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.185395E+00 | grad norm: 0.551 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 242/ 3100 | consumed samples: 495616 | consumed tokens: 1015021568 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.188744E+00 | grad norm: 0.491 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 243/ 3100 | consumed samples: 497664 | consumed tokens: 1019215872 | elapsed time per iteration (s): 148.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.191006E+00 | grad norm: 0.510 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 13.752 | TFLOPs: 140.39 | -[default7]: iteration 244/ 3100 | consumed samples: 499712 | consumed tokens: 1023410176 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.192732E+00 | grad norm: 0.573 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 245/ 3100 | consumed samples: 501760 | consumed tokens: 1027604480 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.174912E+00 | grad norm: 0.495 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 246/ 3100 | consumed samples: 503808 | consumed tokens: 1031798784 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.190127E+00 | grad norm: 0.601 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 247/ 3100 | consumed samples: 505856 | consumed tokens: 1035993088 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.180677E+00 | grad norm: 0.515 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 248/ 3100 | consumed samples: 507904 | consumed tokens: 1040187392 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.193566E+00 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 249/ 3100 | consumed samples: 509952 | consumed tokens: 1044381696 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.191144E+00 | grad norm: 0.487 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 |[default7]: -[default0]:saving checkpoint at iteration 249 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-08 07:53:37,409] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step249 is begin to save! -[default4]:[2022-09-08 07:53:37,424] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_31-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,453] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_45-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,477] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_34-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,424] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_30-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,453] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_44-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_63-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,537] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_28-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,539] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_18-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_71-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_70-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,477] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_35-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,539] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_19-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_49-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_65-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_50-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_10-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_40-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_32-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,559] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_64-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_55-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_56-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_33-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_72-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_59-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_58-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_05-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_62-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_03-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_13-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_57-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_66-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_12-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,537] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_29-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_04-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,572] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_36-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,597] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_06-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,574] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_01-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_16-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,565] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_71_model_states.pt... -[default4]:[2022-09-08 07:53:37,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_71_model_states.pt. -[default4]:[2022-09-08 07:53:37,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_17-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_23-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_38-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_52-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,628] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_14-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,617] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_47-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_43-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_61-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_11-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_53-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,642] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_39-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_22-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_54-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_09-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_69-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_68-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,558] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_48-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_08-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_60-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,571] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_41-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_20-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_51-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_37-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,640] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_46-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,640] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_24-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,628] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_15-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_21-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,597] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_07-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_67-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,619] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_42-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,642] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_27-model_00-model_states.pt... -[default4]:[2022-09-08 07:53:37,645] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_25-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:37,642] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_26-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:40,666] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_30-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:40,666] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_28_model_states.pt... -[default0]:[2022-09-08 07:53:40,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_28_model_states.pt. -[default4]:[2022-09-08 07:53:40,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_09-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:40,820] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_07_model_states.pt... -[default4]:[2022-09-08 07:53:40,826] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_07_model_states.pt. -[default4]:[2022-09-08 07:53:40,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_69-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:40,820] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_67_model_states.pt... -[default4]:[2022-09-08 07:53:40,825] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_67_model_states.pt. -[default0]:[2022-09-08 07:53:40,835] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_58-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:40,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_56_model_states.pt... -[default0]:[2022-09-08 07:53:40,841] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_56_model_states.pt. -[default4]:[2022-09-08 07:53:40,855] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_31-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:40,855] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_29_model_states.pt... -[default4]:[2022-09-08 07:53:40,858] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_29_model_states.pt. -[default0]:[2022-09-08 07:53:40,919] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_72-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:40,920] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_74-model_00-model_states.pt... -[default0]:[2022-09-08 07:53:40,923] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_74-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:40,923] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_70_model_states.pt... -[default0]:[2022-09-08 07:53:40,925] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_70_model_states.pt. -[default4]:[2022-09-08 07:53:40,968] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_23-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:40,968] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_21_model_states.pt... -[default4]:[2022-09-08 07:53:40,987] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_21_model_states.pt. -[default0]:[2022-09-08 07:53:41,000] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_28-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,001] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_26_model_states.pt... -[default0]:[2022-09-08 07:53:41,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_26_model_states.pt. -[default0]:[2022-09-08 07:53:41,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_18-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,005] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_16_model_states.pt... -[default0]:[2022-09-08 07:53:41,008] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_16_model_states.pt. -[default4]:[2022-09-08 07:53:41,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_59-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,051] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_57_model_states.pt... -[default4]:[2022-09-08 07:53:41,059] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_57_model_states.pt. -[default4]:[2022-09-08 07:53:41,110] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_25-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,111] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_23_model_states.pt... -[default4]:[2022-09-08 07:53:41,116] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_23_model_states.pt. -[default0]:[2022-09-08 07:53:41,125] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_22-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,125] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_20_model_states.pt... -[default0]:[2022-09-08 07:53:41,132] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_20_model_states.pt. -[default0]:[2022-09-08 07:53:41,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_68-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,080] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_66_model_states.pt... -[default0]:[2022-09-08 07:53:41,085] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_66_model_states.pt. -[default0]:[2022-09-08 07:53:41,103] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_08-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,103] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_06_model_states.pt... -[default0]:[2022-09-08 07:53:41,106] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_06_model_states.pt. -[default4]:[2022-09-08 07:53:41,136] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_45-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,136] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_43_model_states.pt... -[default4]:[2022-09-08 07:53:41,161] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_43_model_states.pt. -[default4]:[2022-09-08 07:53:41,180] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_29-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,180] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_27_model_states.pt... -[default4]:[2022-09-08 07:53:41,183] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_27_model_states.pt. -[default4]:[2022-09-08 07:53:41,216] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_27-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,217] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_25_model_states.pt... -[default0]:[2022-09-08 07:53:41,143] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_16-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,144] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_14_model_states.pt... -[default0]:[2022-09-08 07:53:41,159] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_14_model_states.pt. -[default0]:[2022-09-08 07:53:41,201] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_44-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,202] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_42_model_states.pt... -[default0]:[2022-09-08 07:53:41,204] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_42_model_states.pt. -[default0]:[2022-09-08 07:53:41,249] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_60-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,250] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_58_model_states.pt... -[default0]:[2022-09-08 07:53:41,200] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_70-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,201] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_68_model_states.pt... -[default0]:[2022-09-08 07:53:41,207] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_68_model_states.pt. -[default4]:[2022-09-08 07:53:41,263] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_19-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,264] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_17_model_states.pt... -[default4]:[2022-09-08 07:53:41,266] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_17_model_states.pt. -[default0]:[2022-09-08 07:53:41,192] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_24-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,192] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_22_model_states.pt... -[default0]:[2022-09-08 07:53:41,199] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_22_model_states.pt. -[default0]:[2022-09-08 07:53:41,227] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_34-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_32_model_states.pt... -[default0]:[2022-09-08 07:53:41,247] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_32_model_states.pt. -[default0]:[2022-09-08 07:53:41,321] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_06-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,321] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_04_model_states.pt... -[default4]:[2022-09-08 07:53:41,237] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_25_model_states.pt. -[default4]:[2022-09-08 07:53:41,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_17-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,263] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_15_model_states.pt... -[default4]:[2022-09-08 07:53:41,265] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_15_model_states.pt. -[default0]:[2022-09-08 07:53:41,317] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_26-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_24_model_states.pt... -[default0]:[2022-09-08 07:53:41,321] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_24_model_states.pt. -[default0]:[2022-09-08 07:53:41,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_58_model_states.pt. -[default4]:[2022-09-08 07:53:41,316] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_71-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_69_model_states.pt... -[default4]:[2022-09-08 07:53:41,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_69_model_states.pt. -[default4]:[2022-09-08 07:53:41,329] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_35-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,330] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_33_model_states.pt... -[default4]:[2022-09-08 07:53:41,332] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_33_model_states.pt. -[default4]:[2022-09-08 07:53:41,371] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_49-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,371] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_47_model_states.pt... -[default0]:[2022-09-08 07:53:41,357] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_10-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,358] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_08_model_states.pt... -[default0]:[2022-09-08 07:53:41,362] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_08_model_states.pt. -[default4]:[2022-09-08 07:53:41,360] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_21-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,361] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_19_model_states.pt... -[default4]:[2022-09-08 07:53:41,366] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_19_model_states.pt. -[default0]:[2022-09-08 07:53:41,375] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_64-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,375] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_62_model_states.pt... -[default0]:[2022-09-08 07:53:41,410] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_66-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,410] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_64_model_states.pt... -[default0]:[2022-09-08 07:53:41,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_04-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,336] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_02_model_states.pt... -[default0]:[2022-09-08 07:53:41,344] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_02_model_states.pt. -[default0]:[2022-09-08 07:53:41,326] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_04_model_states.pt. -[default4]:[2022-09-08 07:53:41,378] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_11-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,378] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_09_model_states.pt... -[default4]:[2022-09-08 07:53:41,396] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_09_model_states.pt. -[default4]:[2022-09-08 07:53:41,386] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_61-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,386] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_59_model_states.pt... -[default4]:[2022-09-08 07:53:41,396] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_59_model_states.pt. -[default0]:[2022-09-08 07:53:41,415] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_20-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,416] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_18_model_states.pt... -[default0]:[2022-09-08 07:53:41,419] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_18_model_states.pt. -[default4]:[2022-09-08 07:53:41,372] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_37-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,372] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_35_model_states.pt... -[default4]:[2022-09-08 07:53:41,388] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_35_model_states.pt. -[default0]:[2022-09-08 07:53:41,400] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_46-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,401] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_44_model_states.pt... -[default0]:[2022-09-08 07:53:41,404] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_44_model_states.pt. -[default4]:[2022-09-08 07:53:41,404] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_47_model_states.pt. -[default4]:[2022-09-08 07:53:41,410] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_65-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,411] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_63_model_states.pt... -[default4]:[2022-09-08 07:53:41,414] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_63_model_states.pt. -[default0]:[2022-09-08 07:53:41,398] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_62_model_states.pt. -[default4]:[2022-09-08 07:53:41,398] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_07-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,398] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_05_model_states.pt... -[default4]:[2022-09-08 07:53:41,400] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_05_model_states.pt. -[default4]:[2022-09-08 07:53:41,464] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_67-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,464] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_65_model_states.pt... -[default4]:[2022-09-08 07:53:41,466] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_65_model_states.pt. -[default4]:[2022-09-08 07:53:41,454] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_05-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,455] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_03_model_states.pt... -[default4]:[2022-09-08 07:53:41,457] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_03_model_states.pt. -[default0]:[2022-09-08 07:53:41,475] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_62-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,476] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_60_model_states.pt... -[default0]:[2022-09-08 07:53:41,497] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_60_model_states.pt. -[default4]:[2022-09-08 07:53:41,505] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_13-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,505] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_11_model_states.pt... -[default4]:[2022-09-08 07:53:41,508] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_11_model_states.pt. -[default0]:[2022-09-08 07:53:41,419] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_64_model_states.pt. -[default0]:[2022-09-08 07:53:41,451] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_12-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,452] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_10_model_states.pt... -[default0]:[2022-09-08 07:53:41,457] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_10_model_states.pt. -[default0]:[2022-09-08 07:53:41,479] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_36-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,479] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_34_model_states.pt... -[default0]:[2022-09-08 07:53:41,490] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_34_model_states.pt. -[default0]:[2022-09-08 07:53:41,499] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_38-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,499] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_36_model_states.pt... -[default0]:[2022-09-08 07:53:41,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_36_model_states.pt. -[default0]:[2022-09-08 07:53:41,491] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_14-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,492] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_12_model_states.pt... -[default0]:[2022-09-08 07:53:41,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_12_model_states.pt. -[default4]:[2022-09-08 07:53:41,490] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_47-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,490] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_45_model_states.pt... -[default4]:[2022-09-08 07:53:41,492] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_45_model_states.pt. -[default4]:[2022-09-08 07:53:41,532] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_63-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,532] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_61_model_states.pt... -[default4]:[2022-09-08 07:53:41,535] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_61_model_states.pt. -[default0]:[2022-09-08 07:53:41,503] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_48-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,504] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_46_model_states.pt... -[default0]:[2022-09-08 07:53:41,509] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_46_model_states.pt. -[default4]:[2022-09-08 07:53:41,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_41-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,528] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_39_model_states.pt... -[default4]:[2022-09-08 07:53:41,547] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_39_model_states.pt. -[default0]:[2022-09-08 07:53:41,547] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_40-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,547] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_38_model_states.pt... -[default0]:[2022-09-08 07:53:41,549] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_38_model_states.pt. -[default4]:[2022-09-08 07:53:41,510] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_15-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,510] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_13_model_states.pt... -[default4]:[2022-09-08 07:53:41,512] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_13_model_states.pt. -[default4]:[2022-09-08 07:53:41,568] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_55-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,568] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_53_model_states.pt... -[default4]:[2022-09-08 07:53:41,581] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_53_model_states.pt. -[default4]:[2022-09-08 07:53:41,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_33-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,577] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_31_model_states.pt... -[default4]:[2022-09-08 07:53:41,580] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_31_model_states.pt. -[default4]:[2022-09-08 07:53:41,566] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_57-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,566] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_55_model_states.pt... -[default4]:[2022-09-08 07:53:41,570] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_55_model_states.pt. -[default4]:[2022-09-08 07:53:41,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_43-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,547] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_41_model_states.pt... -[default4]:[2022-09-08 07:53:41,549] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_41_model_states.pt. -[default4]:[2022-09-08 07:53:41,592] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_39-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,592] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_37_model_states.pt... -[default4]:[2022-09-08 07:53:41,594] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_37_model_states.pt. -[default0]:[2022-09-08 07:53:41,580] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_54-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,581] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_52_model_states.pt... -[default0]:[2022-09-08 07:53:41,583] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_52_model_states.pt. -[default4]:[2022-09-08 07:53:41,654] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_51-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,654] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_49_model_states.pt... -[default4]:[2022-09-08 07:53:41,658] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_49_model_states.pt. -[default0]:[2022-09-08 07:53:41,665] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_50-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,665] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_48_model_states.pt... -[default0]:[2022-09-08 07:53:41,667] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_48_model_states.pt. -[default0]:[2022-09-08 07:53:41,596] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_32-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,597] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_30_model_states.pt... -[default0]:[2022-09-08 07:53:41,598] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_30_model_states.pt. -[default0]:[2022-09-08 07:53:41,639] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_56-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,639] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_54_model_states.pt... -[default0]:[2022-09-08 07:53:41,641] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_54_model_states.pt. -[default0]:[2022-09-08 07:53:41,611] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_42-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,612] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_40_model_states.pt... -[default0]:[2022-09-08 07:53:41,614] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_40_model_states.pt. -[default0]:[2022-09-08 07:53:41,720] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_52-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:41,720] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_50_model_states.pt... -[default0]:[2022-09-08 07:53:41,722] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_50_model_states.pt. -[default4]:[2022-09-08 07:53:41,697] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_53-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:41,697] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_51_model_states.pt... -[default4]:[2022-09-08 07:53:41,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_51_model_states.pt. -[default4]:[2022-09-08 07:53:42,027] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_03-model_00-model_states.pt. -[default4]:[2022-09-08 07:53:42,028] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_01_model_states.pt... -[default4]:[2022-09-08 07:53:42,030] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_01_model_states.pt. -[default0]:[2022-09-08 07:53:43,249] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/layer_01-model_00-model_states.pt. -[default0]:[2022-09-08 07:53:43,250] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_00_model_states.pt -[default0]:[2022-09-08 07:53:43,250] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_00_model_states.pt... -[default0]:[2022-09-08 07:53:43,253] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/mp_rank_00_model_states.pt. -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default2]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default5]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default0]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default4]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default3]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default1]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default7]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default6]:[2022-09-08 07:53:43,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default0]:[2022-09-08 07:53:49,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-08 07:53:49,910] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default3]:[2022-09-08 07:53:50,574] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-08 07:53:50,575] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default2]:[2022-09-08 07:53:50,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-08 07:53:50,561] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default6]:[2022-09-08 07:53:50,628] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-08 07:53:50,628] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default7]:[2022-09-08 07:53:50,776] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-08 07:53:50,776] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default1]:[2022-09-08 07:53:51,093] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-08 07:53:51,094] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default4]:[2022-09-08 07:53:51,306] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-08 07:53:51,306] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default5]:[2022-09-08 07:53:51,603] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-08 07:53:51,604] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default1]:[2022-09-08 07:53:52,321] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-08 07:53:52,322] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default1]:[2022-09-08 07:53:52,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-08 07:53:52,417] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default4]:[2022-09-08 07:53:52,349] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-08 07:53:52,349] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default5]:[2022-09-08 07:53:52,674] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-08 07:53:52,675] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default3]:[2022-09-08 07:53:52,811] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-08 07:53:52,811] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default2]:[2022-09-08 07:53:52,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-08 07:53:52,865] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default0]:[2022-09-08 07:53:52,812] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-08 07:53:52,812] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default0]:[2022-09-08 07:53:52,940] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-08 07:53:52,940] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default0]:[2022-09-08 07:53:53,141] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-08 07:53:53,141] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default2]:[2022-09-08 07:53:53,196] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-08 07:53:53,196] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default3]:[2022-09-08 07:53:53,389] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-08 07:53:53,389] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default1]:[2022-09-08 07:53:53,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-08 07:53:53,407] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default4]:[2022-09-08 07:53:53,549] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-08 07:53:53,549] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default2]:[2022-09-08 07:53:53,545] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-08 07:53:53,545] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default3]:[2022-09-08 07:53:53,584] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-08 07:53:53,584] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default0]:[2022-09-08 07:53:53,619] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-08 07:53:53,619] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default3]:[2022-09-08 07:53:53,668] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-08 07:53:53,668] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default7]:[2022-09-08 07:53:53,646] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-08 07:53:53,646] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default3]:[2022-09-08 07:53:53,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-08 07:53:53,757] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default1]:[2022-09-08 07:53:53,792] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-08 07:53:53,792] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default1]:[2022-09-08 07:53:53,981] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-08 07:53:53,981] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default1]:[2022-09-08 07:53:53,925] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-08 07:53:53,926] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default7]:[2022-09-08 07:53:53,950] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-08 07:53:53,951] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default5]:[2022-09-08 07:53:53,992] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-08 07:53:53,993] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default2]:[2022-09-08 07:53:54,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-08 07:53:54,079] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default3]:[2022-09-08 07:53:54,100] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-08 07:53:54,100] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default7]:[2022-09-08 07:53:54,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-08 07:53:54,079] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default5]:[2022-09-08 07:53:54,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-08 07:53:54,079] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default5]:[2022-09-08 07:53:54,185] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-08 07:53:54,185] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default4]:[2022-09-08 07:53:54,159] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-08 07:53:54,160] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default6]:[2022-09-08 07:53:54,192] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-08 07:53:54,192] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default2]:[2022-09-08 07:53:54,280] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-08 07:53:54,281] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default0]:[2022-09-08 07:53:54,279] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-08 07:53:54,279] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default4]:[2022-09-08 07:53:54,339] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-08 07:53:54,339] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default1]:[2022-09-08 07:53:54,337] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-08 07:53:54,337] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default1]:[2022-09-08 07:53:54,372] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-08 07:53:54,372] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default0]:[2022-09-08 07:53:54,308] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-08 07:53:54,308] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default7]:[2022-09-08 07:53:54,343] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-08 07:53:54,343] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default2]:[2022-09-08 07:53:54,423] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-08 07:53:54,423] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default5]:[2022-09-08 07:53:54,444] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-08 07:53:54,444] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default0]:[2022-09-08 07:53:54,410] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-08 07:53:54,410] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default4]:[2022-09-08 07:53:54,469] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-08 07:53:54,469] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default4]:[2022-09-08 07:53:54,566] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-08 07:53:54,567] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default2]:[2022-09-08 07:53:54,525] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-08 07:53:54,525] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default5]:[2022-09-08 07:53:54,530] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-08 07:53:54,530] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default0]:[2022-09-08 07:53:54,589] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-08 07:53:54,590] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default6]:[2022-09-08 07:53:54,631] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-08 07:53:54,631] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default0]:[2022-09-08 07:53:54,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-08 07:53:54,561] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default6]:[2022-09-08 07:53:54,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-08 07:53:54,579] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default7]:[2022-09-08 07:53:54,586] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-08 07:53:54,586] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default6]:[2022-09-08 07:53:54,583] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-08 07:53:54,583] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default7]:[2022-09-08 07:53:54,648] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-08 07:53:54,648] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default2]:[2022-09-08 07:53:54,687] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-08 07:53:54,687] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default5]:[2022-09-08 07:53:54,749] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-08 07:53:54,749] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default4]:[2022-09-08 07:53:54,660] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-08 07:53:54,660] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default0]:[2022-09-08 07:53:54,677] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-08 07:53:54,677] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default7]:[2022-09-08 07:53:54,723] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-08 07:53:54,723] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default2]:[2022-09-08 07:53:54,709] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-08 07:53:54,710] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default7]:[2022-09-08 07:53:54,725] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-08 07:53:54,725] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default0]:[2022-09-08 07:53:54,730] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-08 07:53:54,730] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default5]:[2022-09-08 07:53:54,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-08 07:53:54,800] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default6]:[2022-09-08 07:53:54,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-08 07:53:54,784] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default2]:[2022-09-08 07:53:54,840] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-08 07:53:54,840] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default7]:[2022-09-08 07:53:54,852] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-08 07:53:54,852] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default7]:[2022-09-08 07:53:54,863] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-08 07:53:54,864] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default4]:[2022-09-08 07:53:54,836] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-08 07:53:54,836] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default5]:[2022-09-08 07:53:54,817] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-08 07:53:54,817] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default6]:[2022-09-08 07:53:54,916] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-08 07:53:54,916] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default2]:[2022-09-08 07:53:54,921] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-08 07:53:54,921] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default5]:[2022-09-08 07:53:54,964] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-08 07:53:54,964] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default4]:[2022-09-08 07:53:54,948] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-08 07:53:54,949] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default4]:[2022-09-08 07:53:55,010] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-08 07:53:55,010] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default7]:[2022-09-08 07:53:54,991] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-08 07:53:54,991] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default6]:[2022-09-08 07:53:54,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-08 07:53:54,994] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default1]:[2022-09-08 07:53:54,963] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-08 07:53:54,963] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default7]:[2022-09-08 07:53:55,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-08 07:53:55,011] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default5]:[2022-09-08 07:53:55,064] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-08 07:53:55,064] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default0]:[2022-09-08 07:53:55,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-08 07:53:55,005] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default3]:[2022-09-08 07:53:55,000] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-08 07:53:55,000] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default3]:[2022-09-08 07:53:55,110] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-08 07:53:55,110] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default5]:[2022-09-08 07:53:55,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-08 07:53:55,070] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default7]:[2022-09-08 07:53:55,110] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-08 07:53:55,110] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default1]:[2022-09-08 07:53:55,081] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-08 07:53:55,081] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default6]:[2022-09-08 07:53:55,154] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-08 07:53:55,154] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default3]:[2022-09-08 07:53:55,110] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-08 07:53:55,110] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default4]:[2022-09-08 07:53:55,107] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-08 07:53:55,107] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default6]:[2022-09-08 07:53:55,129] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-08 07:53:55,130] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default1]:[2022-09-08 07:53:55,146] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-08 07:53:55,146] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default1]:[2022-09-08 07:53:55,202] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-08 07:53:55,202] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default1]:[2022-09-08 07:53:55,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-08 07:53:55,221] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default5]:[2022-09-08 07:53:55,220] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-08 07:53:55,220] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default0]:[2022-09-08 07:53:55,184] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-08 07:53:55,184] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default4]:[2022-09-08 07:53:55,249] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-08 07:53:55,249] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default3]:[2022-09-08 07:53:55,230] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-08 07:53:55,230] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default4]:[2022-09-08 07:53:55,273] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-08 07:53:55,273] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default1]:[2022-09-08 07:53:55,248] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-08 07:53:55,248] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default2]:[2022-09-08 07:53:55,298] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-08 07:53:55,298] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default6]:[2022-09-08 07:53:55,263] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-08 07:53:55,263] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default7]:[2022-09-08 07:53:55,338] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-08 07:53:55,339] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default6]:[2022-09-08 07:53:55,321] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-08 07:53:55,321] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default3]:[2022-09-08 07:53:55,396] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-08 07:53:55,396] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default3]:[2022-09-08 07:53:55,340] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-08 07:53:55,340] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default2]:[2022-09-08 07:53:55,458] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-08 07:53:55,459] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default1]:[2022-09-08 07:53:55,420] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-08 07:53:55,420] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default6]:[2022-09-08 07:53:55,492] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-08 07:53:55,492] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default4]:[2022-09-08 07:53:55,491] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-08 07:53:55,491] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default5]:[2022-09-08 07:53:55,486] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-08 07:53:55,486] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default0]:[2022-09-08 07:53:55,463] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-08 07:53:55,464] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default7]:[2022-09-08 07:53:55,507] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-08 07:53:55,507] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default6]:[2022-09-08 07:53:55,480] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-08 07:53:55,480] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default3]:[2022-09-08 07:53:55,507] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-08 07:53:55,507] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default4]:[2022-09-08 07:53:55,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-08 07:53:55,559] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default2]:[2022-09-08 07:53:55,512] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-08 07:53:55,513] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default5]:[2022-09-08 07:53:55,569] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-08 07:53:55,569] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default1]:[2022-09-08 07:53:55,596] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-08 07:53:55,596] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default2]:[2022-09-08 07:53:55,539] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-08 07:53:55,539] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default7]:[2022-09-08 07:53:55,627] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-08 07:53:55,627] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default5]:[2022-09-08 07:53:55,578] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-08 07:53:55,579] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default7]:[2022-09-08 07:53:55,606] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-08 07:53:55,606] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default0]:[2022-09-08 07:53:55,660] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-08 07:53:55,661] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default1]:[2022-09-08 07:53:55,665] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-08 07:53:55,665] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default6]:[2022-09-08 07:53:55,614] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-08 07:53:55,614] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default3]:[2022-09-08 07:53:55,631] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-08 07:53:55,631] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default7]:[2022-09-08 07:53:55,668] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-08 07:53:55,669] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default1]:[2022-09-08 07:53:55,656] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-08 07:53:55,656] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default4]:[2022-09-08 07:53:55,721] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-08 07:53:55,722] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default6]:[2022-09-08 07:53:55,679] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-08 07:53:55,680] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default6]:[2022-09-08 07:53:55,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-08 07:53:55,685] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default7]:[2022-09-08 07:53:55,676] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-08 07:53:55,676] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default2]:[2022-09-08 07:53:55,715] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-08 07:53:55,715] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default4]:[2022-09-08 07:53:55,803] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-08 07:53:55,803] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default5]:[2022-09-08 07:53:55,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-08 07:53:55,808] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default3]:[2022-09-08 07:53:55,843] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-08 07:53:55,843] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default5]:[2022-09-08 07:53:55,801] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-08 07:53:55,801] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default0]:[2022-09-08 07:53:55,857] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-08 07:53:55,857] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default0]:[2022-09-08 07:53:55,833] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-08 07:53:55,834] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default2]:[2022-09-08 07:53:55,810] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-08 07:53:55,810] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default2]:[2022-09-08 07:53:55,883] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-08 07:53:55,883] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default5]:[2022-09-08 07:53:55,863] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-08 07:53:55,864] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default3]:[2022-09-08 07:53:55,881] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-08 07:53:55,881] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default1]:[2022-09-08 07:53:55,877] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-08 07:53:55,877] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default7]:[2022-09-08 07:53:55,927] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-08 07:53:55,928] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default1]:[2022-09-08 07:53:55,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-08 07:53:55,865] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default6]:[2022-09-08 07:53:55,891] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-08 07:53:55,891] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default0]:[2022-09-08 07:53:55,907] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-08 07:53:55,907] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default1]:[2022-09-08 07:53:55,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-08 07:53:55,947] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default5]:[2022-09-08 07:53:55,958] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-08 07:53:55,958] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default0]:[2022-09-08 07:53:56,092] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-08 07:53:56,092] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default2]:[2022-09-08 07:53:56,104] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-08 07:53:56,104] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default3]:[2022-09-08 07:53:56,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-08 07:53:56,070] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default4]:[2022-09-08 07:53:56,163] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-08 07:53:56,163] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default2]:[2022-09-08 07:53:56,105] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-08 07:53:56,105] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default5]:[2022-09-08 07:53:56,211] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-08 07:53:56,212] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default3]:[2022-09-08 07:53:56,247] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-08 07:53:56,247] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default1]:[2022-09-08 07:53:56,166] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-08 07:53:56,166] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default7]:[2022-09-08 07:53:56,260] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-08 07:53:56,260] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default4]:[2022-09-08 07:53:56,228] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-08 07:53:56,229] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default2]:[2022-09-08 07:53:56,217] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-08 07:53:56,218] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default1]:[2022-09-08 07:53:56,260] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-08 07:53:56,261] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default5]:[2022-09-08 07:53:56,230] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-08 07:53:56,230] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default4]:[2022-09-08 07:53:56,230] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-08 07:53:56,231] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default2]:[2022-09-08 07:53:56,234] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-08 07:53:56,234] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default3]:[2022-09-08 07:53:56,236] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-08 07:53:56,236] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default3]:[2022-09-08 07:53:56,250] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-08 07:53:56,250] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default6]:[2022-09-08 07:53:56,289] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-08 07:53:56,289] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default0]:[2022-09-08 07:53:56,304] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-08 07:53:56,304] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default3]:[2022-09-08 07:53:56,345] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-08 07:53:56,345] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default7]:[2022-09-08 07:53:56,373] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-08 07:53:56,373] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default0]:[2022-09-08 07:53:56,397] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-08 07:53:56,397] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default1]:[2022-09-08 07:53:56,402] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-08 07:53:56,403] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default2]:[2022-09-08 07:53:56,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-08 07:53:56,385] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default5]:[2022-09-08 07:53:56,391] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-08 07:53:56,391] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default1]:[2022-09-08 07:53:56,419] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-08 07:53:56,419] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default2]:[2022-09-08 07:53:56,447] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-08 07:53:56,447] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default7]:[2022-09-08 07:53:56,449] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-08 07:53:56,449] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default4]:[2022-09-08 07:53:56,391] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-08 07:53:56,392] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default3]:[2022-09-08 07:53:56,464] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-08 07:53:56,464] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default2]:[2022-09-08 07:53:56,457] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-08 07:53:56,457] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default6]:[2022-09-08 07:53:56,487] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-08 07:53:56,487] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default4]:[2022-09-08 07:53:56,475] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-08 07:53:56,475] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default6]:[2022-09-08 07:53:56,474] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-08 07:53:56,474] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default4]:[2022-09-08 07:53:56,541] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-08 07:53:56,541] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default3]:[2022-09-08 07:53:56,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-08 07:53:56,559] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default3]:[2022-09-08 07:53:56,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-08 07:53:56,576] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default0]:[2022-09-08 07:53:56,612] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-08 07:53:56,612] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default3]:[2022-09-08 07:53:56,573] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-08 07:53:56,574] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default7]:[2022-09-08 07:53:56,650] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-08 07:53:56,650] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default5]:[2022-09-08 07:53:56,645] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-08 07:53:56,645] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default7]:[2022-09-08 07:53:56,720] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-08 07:53:56,720] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default7]:[2022-09-08 07:53:56,729] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-08 07:53:56,729] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default1]:[2022-09-08 07:53:56,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-08 07:53:56,700] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default6]:[2022-09-08 07:53:56,740] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-08 07:53:56,740] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default6]:[2022-09-08 07:53:56,747] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-08 07:53:56,747] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default6]:[2022-09-08 07:53:56,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-08 07:53:56,770] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default4]:[2022-09-08 07:53:56,794] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-08 07:53:56,794] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default4]:[2022-09-08 07:53:56,721] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-08 07:53:56,722] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default3]:[2022-09-08 07:53:56,777] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-08 07:53:56,777] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default0]:[2022-09-08 07:53:56,822] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-08 07:53:56,822] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default2]:[2022-09-08 07:53:56,847] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-08 07:53:56,847] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default2]:[2022-09-08 07:53:56,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-08 07:53:56,851] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default4]:[2022-09-08 07:53:56,789] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-08 07:53:56,789] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default5]:[2022-09-08 07:53:56,867] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-08 07:53:56,867] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default3]:[2022-09-08 07:53:56,847] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-08 07:53:56,848] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default3]:[2022-09-08 07:53:56,928] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-08 07:53:56,928] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default7]:[2022-09-08 07:53:56,989] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-08 07:53:56,989] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default6]:[2022-09-08 07:53:56,978] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-08 07:53:56,978] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default0]:[2022-09-08 07:53:56,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-08 07:53:56,975] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default4]:[2022-09-08 07:53:56,942] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-08 07:53:56,942] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default0]:[2022-09-08 07:53:57,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-08 07:53:57,025] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default0]:[2022-09-08 07:53:57,073] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-08 07:53:57,074] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default6]:[2022-09-08 07:53:57,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-08 07:53:57,043] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default1]:[2022-09-08 07:53:57,131] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-08 07:53:57,132] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default7]:[2022-09-08 07:53:57,229] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-08 07:53:57,229] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default6]:[2022-09-08 07:53:57,241] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-08 07:53:57,241] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default5]:[2022-09-08 07:53:57,266] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-08 07:53:57,266] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default0]:[2022-09-08 07:53:57,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-08 07:53:57,285] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default5]:[2022-09-08 07:53:57,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-08 07:53:57,351] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default3]:[2022-09-08 07:53:57,357] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-08 07:53:57,357] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default2]:[2022-09-08 07:53:57,473] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-08 07:53:57,473] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default0]:[2022-09-08 07:53:57,494] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-08 07:53:57,494] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default4]:[2022-09-08 07:53:57,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-08 07:53:57,488] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default1]:[2022-09-08 07:53:57,792] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-08 07:53:57,793] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default6]:[2022-09-08 07:53:57,866] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-08 07:53:57,866] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default6]:[2022-09-08 07:53:57,873] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-08 07:53:57,873] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default2]:[2022-09-08 07:53:57,995] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-08 07:53:57,995] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default6]:[2022-09-08 07:53:58,249] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-08 07:53:58,249] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default1]:[2022-09-08 07:53:58,293] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-08 07:53:58,294] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default7]:[2022-09-08 07:53:58,247] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-08 07:53:58,247] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default5]:[2022-09-08 07:53:58,483] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-08 07:53:58,484] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default7]:[2022-09-08 07:53:58,500] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-08 07:53:58,500] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default5]:[2022-09-08 07:53:58,495] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-08 07:53:58,496] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default0]:[2022-09-08 07:53:58,584] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-08 07:53:58,584] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default4]:[2022-09-08 07:53:58,618] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-08 07:53:58,619] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default7]:[2022-09-08 07:53:58,604] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-08 07:53:58,604] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default6]:[2022-09-08 07:53:58,702] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-08 07:53:58,702] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default4]:[2022-09-08 07:53:58,723] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-08 07:53:58,724] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default3]:[2022-09-08 07:53:59,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-08 07:53:59,023] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default5]:[2022-09-08 07:53:59,018] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-08 07:53:59,018] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default5]:[2022-09-08 07:53:59,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-08 07:53:59,047] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default6]:[2022-09-08 07:53:59,071] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-08 07:53:59,071] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default2]:[2022-09-08 07:53:59,164] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-08 07:53:59,164] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default2]:[2022-09-08 07:53:59,170] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-08 07:53:59,171] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default4]:[2022-09-08 07:53:59,226] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-08 07:53:59,226] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default1]:[2022-09-08 07:53:59,220] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-08 07:53:59,220] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default7]:[2022-09-08 07:53:59,258] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-08 07:53:59,258] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default5]:[2022-09-08 07:53:59,257] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-08 07:53:59,257] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default6]:[2022-09-08 07:53:59,256] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-08 07:53:59,256] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default3]:[2022-09-08 07:53:59,449] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-08 07:53:59,449] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default1]:[2022-09-08 07:53:59,543] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-08 07:53:59,543] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default0]:[2022-09-08 07:53:59,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-08 07:53:59,579] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default4]:[2022-09-08 07:53:59,650] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-08 07:53:59,650] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default0]:[2022-09-08 07:53:59,917] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-08 07:53:59,917] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default3]:[2022-09-08 07:54:01,946] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-08 07:54:01,947] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default0]:[2022-09-08 07:54:02,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-08 07:54:02,036] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default2]:[2022-09-08 07:54:02,339] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-08 07:54:02,340] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default3]:[2022-09-08 07:54:03,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-08 07:54:03,647] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default0]:[2022-09-08 07:54:03,697] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-08 07:54:03,697] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default1]:[2022-09-08 07:54:03,979] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-08 07:54:03,979] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default0]:[2022-09-08 07:54:04,202] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-08 07:54:04,211] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default3]:[2022-09-08 07:54:04,818] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-08 07:54:04,818] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default2]:[2022-09-08 07:54:04,884] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-08 07:54:04,884] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default1]:[2022-09-08 07:54:04,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-08 07:54:04,933] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default2]:[2022-09-08 07:54:04,987] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-08 07:54:04,987] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default2]:[2022-09-08 07:54:05,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-08 07:54:05,036] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default1]:[2022-09-08 07:54:05,686] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-08 07:54:05,686] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default3]:[2022-09-08 07:54:06,154] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-08 07:54:06,154] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default5]:[2022-09-08 07:54:07,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-08 07:54:07,156] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default0]:[2022-09-08 07:54:07,930] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-08 07:54:07,930] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default4]:[2022-09-08 07:54:08,889] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-08 07:54:08,889] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default6]:[2022-09-08 07:54:10,301] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-08 07:54:10,301] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default7]:[2022-09-08 07:54:10,494] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-08 07:54:10,494] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default1]:[2022-09-08 07:54:11,387] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-08 07:54:11,387] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default5]:[2022-09-08 07:54:11,508] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-08 07:54:11,508] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default4]:[2022-09-08 07:54:11,604] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-08 07:54:11,604] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default6]:[2022-09-08 07:54:12,526] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-08 07:54:12,526] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default7]:[2022-09-08 07:54:12,574] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-08 07:54:12,574] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default7]:[2022-09-08 07:54:19,954] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-08 07:54:19,954] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default6]:[2022-09-08 07:54:20,296] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-08 07:54:20,296] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default4]:[2022-09-08 07:54:23,744] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-08 07:54:23,744] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]: successfully saved checkpoint at iteration 249 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:time (ms) | save-checkpoint: 46483.13 -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-08 07:54:23,873] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step249/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default6]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default3]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default4]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default5]:[2022-09-08 07:54:23,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default1]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default2]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default0]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]:[2022-09-08 07:54:23,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step249 is ready now! -[default7]: iteration 250/ 3100 | consumed samples: 512000 | consumed tokens: 1048576000 | elapsed time per iteration (s): 187.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.180198E+00 | grad norm: 0.513 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 10.896 | TFLOPs: 111.23 | -[default7]:---------------------------------------------------------------------------------------------------------- -[default7]:validation_pretraining loss at iteration 250 | lm loss value: 2.274159E+00 | lm loss PPL: 9.719741E+00 | -[default7]:---------------------------------------------------------------------------------------------------------- -[default7]: iteration 251/ 3100 | consumed samples: 514048 | consumed tokens: 1052770304 | elapsed time per iteration (s): 187.20 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.175881E+00 | grad norm: 0.571 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 10.940 | TFLOPs: 111.68 | -[default7]: iteration 252/ 3100 | consumed samples: 516096 | consumed tokens: 1056964608 | elapsed time per iteration (s): 147.24 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.178924E+00 | grad norm: 0.608 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 13.909 | TFLOPs: 141.99 | -[default7]: iteration 253/ 3100 | consumed samples: 518144 | consumed tokens: 1061158912 | elapsed time per iteration (s): 144.31 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.189275E+00 | grad norm: 0.489 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.192 | TFLOPs: 144.87 | -[default7]: iteration 254/ 3100 | consumed samples: 520192 | consumed tokens: 1065353216 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.176284E+00 | grad norm: 0.719 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.73 | -[default7]: iteration 255/ 3100 | consumed samples: 522240 | consumed tokens: 1069547520 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.195410E+00 | grad norm: 0.508 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 256/ 3100 | consumed samples: 524288 | consumed tokens: 1073741824 | elapsed time per iteration (s): 142.30 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.184664E+00 | grad norm: 0.501 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.393 | TFLOPs: 146.93 | -[default7]: iteration 257/ 3100 | consumed samples: 526336 | consumed tokens: 1077936128 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.184709E+00 | grad norm: 0.630 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 258/ 3100 | consumed samples: 528384 | consumed tokens: 1082130432 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.171808E+00 | grad norm: 0.549 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 259/ 3100 | consumed samples: 530432 | consumed tokens: 1086324736 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.176807E+00 | grad norm: 0.578 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 260/ 3100 | consumed samples: 532480 | consumed tokens: 1090519040 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.176188E+00 | grad norm: 0.613 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 261/ 3100 | consumed samples: 534528 | consumed tokens: 1094713344 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.172625E+00 | grad norm: 1.208 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 262/ 3100 | consumed samples: 536576 | consumed tokens: 1098907648 | elapsed time per iteration (s): 143.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.179728E+00 | grad norm: 0.560 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.276 | TFLOPs: 145.74 | -[default7]: iteration 263/ 3100 | consumed samples: 538624 | consumed tokens: 1103101952 | elapsed time per iteration (s): 142.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.174523E+00 | grad norm: 0.556 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.334 | TFLOPs: 146.33 | -[default7]: iteration 264/ 3100 | consumed samples: 540672 | consumed tokens: 1107296256 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.183007E+00 | grad norm: 0.496 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 265/ 3100 | consumed samples: 542720 | consumed tokens: 1111490560 | elapsed time per iteration (s): 144.07 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.184755E+00 | grad norm: 1.773 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.216 | TFLOPs: 145.12 | -[default7]: iteration 266/ 3100 | consumed samples: 544768 | consumed tokens: 1115684864 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.184165E+00 | grad norm: 0.667 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 267/ 3100 | consumed samples: 546816 | consumed tokens: 1119879168 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.173316E+00 | grad norm: 0.538 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 268/ 3100 | consumed samples: 548864 | consumed tokens: 1124073472 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.178870E+00 | grad norm: 2.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 269/ 3100 | consumed samples: 550912 | consumed tokens: 1128267776 | elapsed time per iteration (s): 140.98 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.166474E+00 | grad norm: 0.650 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.526 | TFLOPs: 148.29 | -[default7]: iteration 270/ 3100 | consumed samples: 552960 | consumed tokens: 1132462080 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.173785E+00 | grad norm: 0.608 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 271/ 3100 | consumed samples: 555008 | consumed tokens: 1136656384 | elapsed time per iteration (s): 143.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.172234E+00 | grad norm: 0.753 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.239 | TFLOPs: 145.36 | -[default7]: iteration 272/ 3100 | consumed samples: 557056 | consumed tokens: 1140850688 | elapsed time per iteration (s): 142.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.174999E+00 | grad norm: 0.482 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.383 | TFLOPs: 146.83 | -[default7]: iteration 273/ 3100 | consumed samples: 559104 | consumed tokens: 1145044992 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.186187E+00 | grad norm: 0.614 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 274/ 3100 | consumed samples: 561152 | consumed tokens: 1149239296 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.159837E+00 | grad norm: 0.510 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 275/ 3100 | consumed samples: 563200 | consumed tokens: 1153433600 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.159787E+00 | grad norm: 0.452 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 276/ 3100 | consumed samples: 565248 | consumed tokens: 1157627904 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.181321E+00 | grad norm: 0.482 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 277/ 3100 | consumed samples: 567296 | consumed tokens: 1161822208 | elapsed time per iteration (s): 140.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.171361E+00 | grad norm: 0.519 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.528 | TFLOPs: 148.31 | -[default7]: iteration 278/ 3100 | consumed samples: 569344 | consumed tokens: 1166016512 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.165603E+00 | grad norm: 0.505 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 279/ 3100 | consumed samples: 571392 | consumed tokens: 1170210816 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.166553E+00 | grad norm: 0.482 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 280/ 3100 | consumed samples: 573440 | consumed tokens: 1174405120 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.176668E+00 | grad norm: 0.552 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 281/ 3100 | consumed samples: 575488 | consumed tokens: 1178599424 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.164205E+00 | grad norm: 0.485 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 282/ 3100 | consumed samples: 577536 | consumed tokens: 1182793728 | elapsed time per iteration (s): 142.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.155347E+00 | grad norm: 0.466 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.421 | TFLOPs: 147.22 | -[default7]: iteration 283/ 3100 | consumed samples: 579584 | consumed tokens: 1186988032 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.157313E+00 | grad norm: 0.474 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 284/ 3100 | consumed samples: 581632 | consumed tokens: 1191182336 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.155542E+00 | grad norm: 0.446 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.54 | -[default7]: iteration 285/ 3100 | consumed samples: 583680 | consumed tokens: 1195376640 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.167163E+00 | grad norm: 0.465 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.48 | -[default7]: iteration 286/ 3100 | consumed samples: 585728 | consumed tokens: 1199570944 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.156306E+00 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 287/ 3100 | consumed samples: 587776 | consumed tokens: 1203765248 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.174853E+00 | grad norm: 0.448 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 288/ 3100 | consumed samples: 589824 | consumed tokens: 1207959552 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.140602E+00 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 289/ 3100 | consumed samples: 591872 | consumed tokens: 1212153856 | elapsed time per iteration (s): 140.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.164434E+00 | grad norm: 0.500 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.626 | TFLOPs: 149.31 | -[default7]: iteration 290/ 3100 | consumed samples: 593920 | consumed tokens: 1216348160 | elapsed time per iteration (s): 145.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.152800E+00 | grad norm: 0.459 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.084 | TFLOPs: 143.78 | -[default7]: iteration 291/ 3100 | consumed samples: 595968 | consumed tokens: 1220542464 | elapsed time per iteration (s): 146.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.160845E+00 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 13.996 | TFLOPs: 142.88 | -[default7]: iteration 292/ 3100 | consumed samples: 598016 | consumed tokens: 1224736768 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.155729E+00 | grad norm: 0.475 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 293/ 3100 | consumed samples: 600064 | consumed tokens: 1228931072 | elapsed time per iteration (s): 141.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.158278E+00 | grad norm: 0.524 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.485 | TFLOPs: 147.87 | -[default7]: iteration 294/ 3100 | consumed samples: 602112 | consumed tokens: 1233125376 | elapsed time per iteration (s): 140.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.151618E+00 | grad norm: 0.451 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.566 | TFLOPs: 148.69 | -[default7]: iteration 295/ 3100 | consumed samples: 604160 | consumed tokens: 1237319680 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.158260E+00 | grad norm: 0.542 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 296/ 3100 | consumed samples: 606208 | consumed tokens: 1241513984 | elapsed time per iteration (s): 154.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.149587E+00 | grad norm: 0.464 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 13.233 | TFLOPs: 135.08 | -[default7]: iteration 297/ 3100 | consumed samples: 608256 | consumed tokens: 1245708288 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.151549E+00 | grad norm: 0.466 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 298/ 3100 | consumed samples: 610304 | consumed tokens: 1249902592 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.144989E+00 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 299/ 3100 | consumed samples: 612352 | consumed tokens: 1254096896 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.145907E+00 | grad norm: 0.487 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 300/ 3100 | consumed samples: 614400 | consumed tokens: 1258291200 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.161359E+00 | grad norm: 0.519 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 301/ 3100 | consumed samples: 616448 | consumed tokens: 1262485504 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.151229E+00 | grad norm: 0.518 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 302/ 3100 | consumed samples: 618496 | consumed tokens: 1266679808 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.144911E+00 | grad norm: 0.464 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 303/ 3100 | consumed samples: 620544 | consumed tokens: 1270874112 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.158231E+00 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 304/ 3100 | consumed samples: 622592 | consumed tokens: 1275068416 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.148294E+00 | grad norm: 0.572 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 305/ 3100 | consumed samples: 624640 | consumed tokens: 1279262720 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.154390E+00 | grad norm: 0.470 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 306/ 3100 | consumed samples: 626688 | consumed tokens: 1283457024 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.145318E+00 | grad norm: 0.464 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 307/ 3100 | consumed samples: 628736 | consumed tokens: 1287651328 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.122244E+00 | grad norm: 0.520 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 308/ 3100 | consumed samples: 630784 | consumed tokens: 1291845632 | elapsed time per iteration (s): 140.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.169281E+00 | grad norm: 0.473 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.568 | TFLOPs: 148.72 | -[default7]: iteration 309/ 3100 | consumed samples: 632832 | consumed tokens: 1296039936 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.143699E+00 | grad norm: 0.512 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 310/ 3100 | consumed samples: 634880 | consumed tokens: 1300234240 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.146071E+00 | grad norm: 0.466 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 311/ 3100 | consumed samples: 636928 | consumed tokens: 1304428544 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.143831E+00 | grad norm: 0.501 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 312/ 3100 | consumed samples: 638976 | consumed tokens: 1308622848 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.146141E+00 | grad norm: 0.499 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 313/ 3100 | consumed samples: 641024 | consumed tokens: 1312817152 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.145675E+00 | grad norm: 0.448 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 314/ 3100 | consumed samples: 643072 | consumed tokens: 1317011456 | elapsed time per iteration (s): 140.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.140470E+00 | grad norm: 0.522 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.569 | TFLOPs: 148.72 | -[default7]: iteration 315/ 3100 | consumed samples: 645120 | consumed tokens: 1321205760 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.138327E+00 | grad norm: 0.454 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 316/ 3100 | consumed samples: 647168 | consumed tokens: 1325400064 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.142691E+00 | grad norm: 0.503 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 317/ 3100 | consumed samples: 649216 | consumed tokens: 1329594368 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.134046E+00 | grad norm: 0.460 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 318/ 3100 | consumed samples: 651264 | consumed tokens: 1333788672 | elapsed time per iteration (s): 140.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.118758E+00 | grad norm: 0.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.587 | TFLOPs: 148.91 | -[default7]: iteration 319/ 3100 | consumed samples: 653312 | consumed tokens: 1337982976 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.132419E+00 | grad norm: 0.434 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 | -[default7]: iteration 320/ 3100 | consumed samples: 655360 | consumed tokens: 1342177280 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.131887E+00 | grad norm: 0.625 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 321/ 3100 | consumed samples: 657408 | consumed tokens: 1346371584 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.140998E+00 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 322/ 3100 | consumed samples: 659456 | consumed tokens: 1350565888 | elapsed time per iteration (s): 141.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.140716E+00 | grad norm: 0.474 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.429 | TFLOPs: 147.30 | -[default7]: iteration 323/ 3100 | consumed samples: 661504 | consumed tokens: 1354760192 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.131970E+00 | grad norm: 0.484 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 324/ 3100 | consumed samples: 663552 | consumed tokens: 1358954496 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.147591E+00 | grad norm: 0.504 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 325/ 3100 | consumed samples: 665600 | consumed tokens: 1363148800 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.128777E+00 | grad norm: 0.547 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 326/ 3100 | consumed samples: 667648 | consumed tokens: 1367343104 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.134371E+00 | grad norm: 0.453 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 327/ 3100 | consumed samples: 669696 | consumed tokens: 1371537408 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.156462E+00 | grad norm: 0.590 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 328/ 3100 | consumed samples: 671744 | consumed tokens: 1375731712 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.134589E+00 | grad norm: 0.460 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 329/ 3100 | consumed samples: 673792 | consumed tokens: 1379926016 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.119727E+00 | grad norm: 0.473 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 330/ 3100 | consumed samples: 675840 | consumed tokens: 1384120320 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.139506E+00 | grad norm: 0.642 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 331/ 3100 | consumed samples: 677888 | consumed tokens: 1388314624 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.122085E+00 | grad norm: 0.492 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 332/ 3100 | consumed samples: 679936 | consumed tokens: 1392508928 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.131431E+00 | grad norm: 0.529 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 333/ 3100 | consumed samples: 681984 | consumed tokens: 1396703232 | elapsed time per iteration (s): 141.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.135427E+00 | grad norm: 0.652 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.488 | TFLOPs: 147.90 | -[default7]: iteration 334/ 3100 | consumed samples: 684032 | consumed tokens: 1400897536 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.138903E+00 | grad norm: 0.445 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 335/ 3100 | consumed samples: 686080 | consumed tokens: 1405091840 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.110568E+00 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.35 | -[default7]: iteration 336/ 3100 | consumed samples: 688128 | consumed tokens: 1409286144 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.123527E+00 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 337/ 3100 | consumed samples: 690176 | consumed tokens: 1413480448 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.119598E+00 | grad norm: 0.486 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 338/ 3100 | consumed samples: 692224 | consumed tokens: 1417674752 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.122376E+00 | grad norm: 0.457 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 339/ 3100 | consumed samples: 694272 | consumed tokens: 1421869056 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.135492E+00 | grad norm: 0.489 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 340/ 3100 | consumed samples: 696320 | consumed tokens: 1426063360 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.118042E+00 | grad norm: 0.470 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 341/ 3100 | consumed samples: 698368 | consumed tokens: 1430257664 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.114972E+00 | grad norm: 0.495 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 342/ 3100 | consumed samples: 700416 | consumed tokens: 1434451968 | elapsed time per iteration (s): 140.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.143852E+00 | grad norm: 0.493 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.569 | TFLOPs: 148.73 | -[default7]: iteration 343/ 3100 | consumed samples: 702464 | consumed tokens: 1438646272 | elapsed time per iteration (s): 140.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.124752E+00 | grad norm: 0.544 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.532 | TFLOPs: 148.35 | -[default7]: iteration 344/ 3100 | consumed samples: 704512 | consumed tokens: 1442840576 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.114355E+00 | grad norm: 0.502 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 345/ 3100 | consumed samples: 706560 | consumed tokens: 1447034880 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.109742E+00 | grad norm: 0.455 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 346/ 3100 | consumed samples: 708608 | consumed tokens: 1451229184 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.114647E+00 | grad norm: 0.554 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 347/ 3100 | consumed samples: 710656 | consumed tokens: 1455423488 | elapsed time per iteration (s): 140.26 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.123215E+00 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.601 | TFLOPs: 149.05 | -[default7]: iteration 348/ 3100 | consumed samples: 712704 | consumed tokens: 1459617792 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.143628E+00 | grad norm: 0.514 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 349/ 3100 | consumed samples: 714752 | consumed tokens: 1463812096 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.119876E+00 | grad norm: 0.477 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.74 | -[default7]: iteration 350/ 3100 | consumed samples: 716800 | consumed tokens: 1468006400 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.105029E+00 | grad norm: 0.469 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 351/ 3100 | consumed samples: 718848 | consumed tokens: 1472200704 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.127620E+00 | grad norm: 0.557 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 352/ 3100 | consumed samples: 720896 | consumed tokens: 1476395008 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.116540E+00 | grad norm: 0.490 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 353/ 3100 | consumed samples: 722944 | consumed tokens: 1480589312 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.116872E+00 | grad norm: 0.504 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 354/ 3100 | consumed samples: 724992 | consumed tokens: 1484783616 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.127033E+00 | grad norm: 0.532 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 355/ 3100 | consumed samples: 727040 | consumed tokens: 1488977920 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.124233E+00 | grad norm: 0.517 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 356/ 3100 | consumed samples: 729088 | consumed tokens: 1493172224 | elapsed time per iteration (s): 140.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.111251E+00 | grad norm: 0.531 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.594 | TFLOPs: 148.98 | -[default7]: iteration 357/ 3100 | consumed samples: 731136 | consumed tokens: 1497366528 | elapsed time per iteration (s): 139.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.130126E+00 | grad norm: 0.537 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.649 | TFLOPs: 149.54 | -[default7]: iteration 358/ 3100 | consumed samples: 733184 | consumed tokens: 1501560832 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.121806E+00 | grad norm: 0.466 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 359/ 3100 | consumed samples: 735232 | consumed tokens: 1505755136 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.104037E+00 | grad norm: 0.553 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 360/ 3100 | consumed samples: 737280 | consumed tokens: 1509949440 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.119420E+00 | grad norm: 0.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 361/ 3100 | consumed samples: 739328 | consumed tokens: 1514143744 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.115443E+00 | grad norm: 0.627 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.75 | -[default7]: iteration 362/ 3100 | consumed samples: 741376 | consumed tokens: 1518338048 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.108521E+00 | grad norm: 0.560 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 363/ 3100 | consumed samples: 743424 | consumed tokens: 1522532352 | elapsed time per iteration (s): 143.34 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.115987E+00 | grad norm: 0.498 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.288 | TFLOPs: 145.86 | -[default7]: iteration 364/ 3100 | consumed samples: 745472 | consumed tokens: 1526726656 | elapsed time per iteration (s): 140.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.107506E+00 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.570 | TFLOPs: 148.74 | -[default7]: iteration 365/ 3100 | consumed samples: 747520 | consumed tokens: 1530920960 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.105307E+00 | grad norm: 0.452 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 366/ 3100 | consumed samples: 749568 | consumed tokens: 1535115264 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.117522E+00 | grad norm: 0.498 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 367/ 3100 | consumed samples: 751616 | consumed tokens: 1539309568 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.099238E+00 | grad norm: 3.070 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 368/ 3100 | consumed samples: 753664 | consumed tokens: 1543503872 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.110788E+00 | grad norm: 0.556 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 369/ 3100 | consumed samples: 755712 | consumed tokens: 1547698176 | elapsed time per iteration (s): 140.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.121269E+00 | grad norm: 1.295 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.542 | TFLOPs: 148.45 | -[default7]: iteration 370/ 3100 | consumed samples: 757760 | consumed tokens: 1551892480 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.108963E+00 | grad norm: 1.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 371/ 3100 | consumed samples: 759808 | consumed tokens: 1556086784 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.111805E+00 | grad norm: 0.528 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 372/ 3100 | consumed samples: 761856 | consumed tokens: 1560281088 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.104306E+00 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 373/ 3100 | consumed samples: 763904 | consumed tokens: 1564475392 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.102992E+00 | grad norm: 0.557 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 374/ 3100 | consumed samples: 765952 | consumed tokens: 1568669696 | elapsed time per iteration (s): 140.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.120290E+00 | grad norm: 0.482 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.570 | TFLOPs: 148.74 | -[default7]: iteration 375/ 3100 | consumed samples: 768000 | consumed tokens: 1572864000 | elapsed time per iteration (s): 140.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.106447E+00 | grad norm: 0.488 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.553 | TFLOPs: 148.57 | -[default7]: iteration 376/ 3100 | consumed samples: 770048 | consumed tokens: 1577058304 | elapsed time per iteration (s): 142.01 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.113233E+00 | grad norm: 0.560 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.422 | TFLOPs: 147.22 | -[default7]: iteration 377/ 3100 | consumed samples: 772096 | consumed tokens: 1581252608 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.107117E+00 | grad norm: 0.485 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 378/ 3100 | consumed samples: 774144 | consumed tokens: 1585446912 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.117589E+00 | grad norm: 0.454 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 379/ 3100 | consumed samples: 776192 | consumed tokens: 1589641216 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.090935E+00 | grad norm: 0.490 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 380/ 3100 | consumed samples: 778240 | consumed tokens: 1593835520 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.094735E+00 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 381/ 3100 | consumed samples: 780288 | consumed tokens: 1598029824 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.105106E+00 | grad norm: 0.992 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 382/ 3100 | consumed samples: 782336 | consumed tokens: 1602224128 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.108890E+00 | grad norm: 0.465 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 383/ 3100 | consumed samples: 784384 | consumed tokens: 1606418432 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.095739E+00 | grad norm: 0.498 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 384/ 3100 | consumed samples: 786432 | consumed tokens: 1610612736 | elapsed time per iteration (s): 140.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.095117E+00 | grad norm: 0.449 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.542 | TFLOPs: 148.45 | -[default7]: iteration 385/ 3100 | consumed samples: 788480 | consumed tokens: 1614807040 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.107075E+00 | grad norm: 0.469 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.54 | -[default7]: iteration 386/ 3100 | consumed samples: 790528 | consumed tokens: 1619001344 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.086869E+00 | grad norm: 0.485 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 387/ 3100 | consumed samples: 792576 | consumed tokens: 1623195648 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.092280E+00 | grad norm: 0.497 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 388/ 3100 | consumed samples: 794624 | consumed tokens: 1627389952 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.106120E+00 | grad norm: 0.446 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 389/ 3100 | consumed samples: 796672 | consumed tokens: 1631584256 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.095424E+00 | grad norm: 0.461 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 390/ 3100 | consumed samples: 798720 | consumed tokens: 1635778560 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.100818E+00 | grad norm: 0.454 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 391/ 3100 | consumed samples: 800768 | consumed tokens: 1639972864 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.112707E+00 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 392/ 3100 | consumed samples: 802816 | consumed tokens: 1644167168 | elapsed time per iteration (s): 140.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.101185E+00 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.581 | TFLOPs: 148.85 | -[default7]: iteration 393/ 3100 | consumed samples: 804864 | consumed tokens: 1648361472 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.102564E+00 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 394/ 3100 | consumed samples: 806912 | consumed tokens: 1652555776 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.111630E+00 | grad norm: 0.469 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 395/ 3100 | consumed samples: 808960 | consumed tokens: 1656750080 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.087429E+00 | grad norm: 0.482 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 396/ 3100 | consumed samples: 811008 | consumed tokens: 1660944384 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.082388E+00 | grad norm: 0.702 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 397/ 3100 | consumed samples: 813056 | consumed tokens: 1665138688 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.091794E+00 | grad norm: 0.507 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 398/ 3100 | consumed samples: 815104 | consumed tokens: 1669332992 | elapsed time per iteration (s): 141.38 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.083789E+00 | grad norm: 0.508 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.485 | TFLOPs: 147.87 | -[default7]: iteration 399/ 3100 | consumed samples: 817152 | consumed tokens: 1673527296 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.088295E+00 | grad norm: 0.658 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 400/ 3100 | consumed samples: 819200 | consumed tokens: 1677721600 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.096523E+00 | grad norm: 0.449 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 401/ 3100 | consumed samples: 821248 | consumed tokens: 1681915904 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.107410E+00 | grad norm: 0.480 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 402/ 3100 | consumed samples: 823296 | consumed tokens: 1686110208 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.086815E+00 | grad norm: 0.461 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 403/ 3100 | consumed samples: 825344 | consumed tokens: 1690304512 | elapsed time per iteration (s): 140.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.096372E+00 | grad norm: 0.448 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.540 | TFLOPs: 148.43 | -[default7]: iteration 404/ 3100 | consumed samples: 827392 | consumed tokens: 1694498816 | elapsed time per iteration (s): 140.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.092389E+00 | grad norm: 0.498 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.536 | TFLOPs: 148.39 | -[default7]: iteration 405/ 3100 | consumed samples: 829440 | consumed tokens: 1698693120 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.087955E+00 | grad norm: 0.479 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 406/ 3100 | consumed samples: 831488 | consumed tokens: 1702887424 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.075851E+00 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.581 | TFLOPs: 148.85 | -[default7]: iteration 407/ 3100 | consumed samples: 833536 | consumed tokens: 1707081728 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.109478E+00 | grad norm: 1.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 408/ 3100 | consumed samples: 835584 | consumed tokens: 1711276032 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.098321E+00 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 409/ 3100 | consumed samples: 837632 | consumed tokens: 1715470336 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.105246E+00 | grad norm: 0.482 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 410/ 3100 | consumed samples: 839680 | consumed tokens: 1719664640 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.089778E+00 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 411/ 3100 | consumed samples: 841728 | consumed tokens: 1723858944 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.072554E+00 | grad norm: 0.494 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 412/ 3100 | consumed samples: 843776 | consumed tokens: 1728053248 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.090216E+00 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 413/ 3100 | consumed samples: 845824 | consumed tokens: 1732247552 | elapsed time per iteration (s): 140.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.084448E+00 | grad norm: 81.580 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.557 | TFLOPs: 148.60 | -[default7]: iteration 414/ 3100 | consumed samples: 847872 | consumed tokens: 1736441856 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.102282E+00 | grad norm: 0.667 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 415/ 3100 | consumed samples: 849920 | consumed tokens: 1740636160 | elapsed time per iteration (s): 141.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.082766E+00 | grad norm: 0.458 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.429 | TFLOPs: 147.30 | -[default7]: iteration 416/ 3100 | consumed samples: 851968 | consumed tokens: 1744830464 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.090503E+00 | grad norm: 0.626 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 417/ 3100 | consumed samples: 854016 | consumed tokens: 1749024768 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.080737E+00 | grad norm: 0.480 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 418/ 3100 | consumed samples: 856064 | consumed tokens: 1753219072 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.093361E+00 | grad norm: 0.516 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 419/ 3100 | consumed samples: 858112 | consumed tokens: 1757413376 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.083243E+00 | grad norm: 0.461 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 420/ 3100 | consumed samples: 860160 | consumed tokens: 1761607680 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.096908E+00 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 421/ 3100 | consumed samples: 862208 | consumed tokens: 1765801984 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.093266E+00 | grad norm: 0.447 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 422/ 3100 | consumed samples: 864256 | consumed tokens: 1769996288 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.091096E+00 | grad norm: 1.605 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 423/ 3100 | consumed samples: 866304 | consumed tokens: 1774190592 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.072680E+00 | grad norm: 0.504 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 424/ 3100 | consumed samples: 868352 | consumed tokens: 1778384896 | elapsed time per iteration (s): 140.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.101993E+00 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.588 | TFLOPs: 148.92 | -[default7]: iteration 425/ 3100 | consumed samples: 870400 | consumed tokens: 1782579200 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.076478E+00 | grad norm: 0.466 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 426/ 3100 | consumed samples: 872448 | consumed tokens: 1786773504 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.076886E+00 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 427/ 3100 | consumed samples: 874496 | consumed tokens: 1790967808 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.094347E+00 | grad norm: 0.495 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 428/ 3100 | consumed samples: 876544 | consumed tokens: 1795162112 | elapsed time per iteration (s): 142.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.077897E+00 | grad norm: 0.445 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.383 | TFLOPs: 146.83 | -[default7]: iteration 429/ 3100 | consumed samples: 878592 | consumed tokens: 1799356416 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.077335E+00 | grad norm: 0.451 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 430/ 3100 | consumed samples: 880640 | consumed tokens: 1803550720 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.084246E+00 | grad norm: 0.492 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 431/ 3100 | consumed samples: 882688 | consumed tokens: 1807745024 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.070956E+00 | grad norm: 0.446 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 432/ 3100 | consumed samples: 884736 | consumed tokens: 1811939328 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.095393E+00 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 433/ 3100 | consumed samples: 886784 | consumed tokens: 1816133632 | elapsed time per iteration (s): 140.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.073813E+00 | grad norm: 0.434 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.533 | TFLOPs: 148.36 | -[default7]: iteration 434/ 3100 | consumed samples: 888832 | consumed tokens: 1820327936 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.067949E+00 | grad norm: 0.447 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.54 | -[default7]: iteration 435/ 3100 | consumed samples: 890880 | consumed tokens: 1824522240 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.082224E+00 | grad norm: 0.484 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 436/ 3100 | consumed samples: 892928 | consumed tokens: 1828716544 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.070879E+00 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.38 | -[default7]: iteration 437/ 3100 | consumed samples: 894976 | consumed tokens: 1832910848 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.067401E+00 | grad norm: 0.593 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 438/ 3100 | consumed samples: 897024 | consumed tokens: 1837105152 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.066178E+00 | grad norm: 0.493 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 439/ 3100 | consumed samples: 899072 | consumed tokens: 1841299456 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.067053E+00 | grad norm: 0.516 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 440/ 3100 | consumed samples: 901120 | consumed tokens: 1845493760 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.080244E+00 | grad norm: 0.481 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 441/ 3100 | consumed samples: 903168 | consumed tokens: 1849688064 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.080922E+00 | grad norm: 0.454 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 442/ 3100 | consumed samples: 905216 | consumed tokens: 1853882368 | elapsed time per iteration (s): 140.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.075594E+00 | grad norm: 0.516 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.588 | TFLOPs: 148.92 | -[default7]: iteration 443/ 3100 | consumed samples: 907264 | consumed tokens: 1858076672 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.081147E+00 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.427 | TFLOPs: 147.27 | -[default7]: iteration 444/ 3100 | consumed samples: 909312 | consumed tokens: 1862270976 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.057900E+00 | grad norm: 0.580 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 445/ 3100 | consumed samples: 911360 | consumed tokens: 1866465280 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.061372E+00 | grad norm: 0.460 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 446/ 3100 | consumed samples: 913408 | consumed tokens: 1870659584 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.074309E+00 | grad norm: 0.513 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 447/ 3100 | consumed samples: 915456 | consumed tokens: 1874853888 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.064823E+00 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 448/ 3100 | consumed samples: 917504 | consumed tokens: 1879048192 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.047346E+00 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.45 | -[default7]: iteration 449/ 3100 | consumed samples: 919552 | consumed tokens: 1883242496 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.074328E+00 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 450/ 3100 | consumed samples: 921600 | consumed tokens: 1887436800 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.064013E+00 | grad norm: 0.589 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 451/ 3100 | consumed samples: 923648 | consumed tokens: 1891631104 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.053654E+00 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 452/ 3100 | consumed samples: 925696 | consumed tokens: 1895825408 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.076205E+00 | grad norm: 0.475 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 453/ 3100 | consumed samples: 927744 | consumed tokens: 1900019712 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.046520E+00 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 454/ 3100 | consumed samples: 929792 | consumed tokens: 1904214016 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.062738E+00 | grad norm: 0.557 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.49 | -[default7]: iteration 455/ 3100 | consumed samples: 931840 | consumed tokens: 1908408320 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.061006E+00 | grad norm: 0.453 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.38 | -[default7]: iteration 456/ 3100 | consumed samples: 933888 | consumed tokens: 1912602624 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.062370E+00 | grad norm: 0.492 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 457/ 3100 | consumed samples: 935936 | consumed tokens: 1916796928 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.083495E+00 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 458/ 3100 | consumed samples: 937984 | consumed tokens: 1920991232 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.056751E+00 | grad norm: 0.445 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 459/ 3100 | consumed samples: 940032 | consumed tokens: 1925185536 | elapsed time per iteration (s): 140.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.066480E+00 | grad norm: 0.478 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.581 | TFLOPs: 148.85 | -[default7]: iteration 460/ 3100 | consumed samples: 942080 | consumed tokens: 1929379840 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.036331E+00 | grad norm: 0.619 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 461/ 3100 | consumed samples: 944128 | consumed tokens: 1933574144 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.064803E+00 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 462/ 3100 | consumed samples: 946176 | consumed tokens: 1937768448 | elapsed time per iteration (s): 139.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.054498E+00 | grad norm: 0.481 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.648 | TFLOPs: 149.53 | -[default7]: iteration 463/ 3100 | consumed samples: 948224 | consumed tokens: 1941962752 | elapsed time per iteration (s): 139.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.065519E+00 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.632 | TFLOPs: 149.37 | -[default7]: iteration 464/ 3100 | consumed samples: 950272 | consumed tokens: 1946157056 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.041342E+00 | grad norm: 0.503 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 465/ 3100 | consumed samples: 952320 | consumed tokens: 1950351360 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.047861E+00 | grad norm: 0.443 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 466/ 3100 | consumed samples: 954368 | consumed tokens: 1954545664 | elapsed time per iteration (s): 140.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.053452E+00 | grad norm: 0.586 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.544 | TFLOPs: 148.47 | -[default7]: iteration 467/ 3100 | consumed samples: 956416 | consumed tokens: 1958739968 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.061106E+00 | grad norm: 0.469 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 468/ 3100 | consumed samples: 958464 | consumed tokens: 1962934272 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.066727E+00 | grad norm: 0.474 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 469/ 3100 | consumed samples: 960512 | consumed tokens: 1967128576 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.055323E+00 | grad norm: 0.450 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 470/ 3100 | consumed samples: 962560 | consumed tokens: 1971322880 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.052683E+00 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 471/ 3100 | consumed samples: 964608 | consumed tokens: 1975517184 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.059497E+00 | grad norm: 0.470 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 472/ 3100 | consumed samples: 966656 | consumed tokens: 1979711488 | elapsed time per iteration (s): 140.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.070358E+00 | grad norm: 1.334 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.572 | TFLOPs: 148.76 | -[default7]: iteration 473/ 3100 | consumed samples: 968704 | consumed tokens: 1983905792 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.063440E+00 | grad norm: 0.436 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 474/ 3100 | consumed samples: 970752 | consumed tokens: 1988100096 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.054245E+00 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 475/ 3100 | consumed samples: 972800 | consumed tokens: 1992294400 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.053300E+00 | grad norm: 0.437 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 476/ 3100 | consumed samples: 974848 | consumed tokens: 1996488704 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.062515E+00 | grad norm: 0.458 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 477/ 3100 | consumed samples: 976896 | consumed tokens: 2000683008 | elapsed time per iteration (s): 142.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.064769E+00 | grad norm: 2.072 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.423 | TFLOPs: 147.24 | -[default7]: iteration 478/ 3100 | consumed samples: 978944 | consumed tokens: 2004877312 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.049946E+00 | grad norm: 0.451 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 479/ 3100 | consumed samples: 980992 | consumed tokens: 2009071616 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.047569E+00 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 480/ 3100 | consumed samples: 983040 | consumed tokens: 2013265920 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.054322E+00 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 481/ 3100 | consumed samples: 985088 | consumed tokens: 2017460224 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.052581E+00 | grad norm: 0.423 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 482/ 3100 | consumed samples: 987136 | consumed tokens: 2021654528 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.061928E+00 | grad norm: 0.443 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 483/ 3100 | consumed samples: 989184 | consumed tokens: 2025848832 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.045121E+00 | grad norm: 0.458 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 484/ 3100 | consumed samples: 991232 | consumed tokens: 2030043136 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.057921E+00 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 485/ 3100 | consumed samples: 993280 | consumed tokens: 2034237440 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.054802E+00 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 486/ 3100 | consumed samples: 995328 | consumed tokens: 2038431744 | elapsed time per iteration (s): 142.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.044811E+00 | grad norm: 0.450 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.419 | TFLOPs: 147.20 | -[default7]: iteration 487/ 3100 | consumed samples: 997376 | consumed tokens: 2042626048 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.065208E+00 | grad norm: 0.457 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 488/ 3100 | consumed samples: 999424 | consumed tokens: 2046820352 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.047825E+00 | grad norm: 0.561 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 489/ 3100 | consumed samples: 1001472 | consumed tokens: 2051014656 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.059938E+00 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 490/ 3100 | consumed samples: 1003520 | consumed tokens: 2055208960 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.053261E+00 | grad norm: 0.506 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 491/ 3100 | consumed samples: 1005568 | consumed tokens: 2059403264 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.037044E+00 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.26 | -[default7]: iteration 492/ 3100 | consumed samples: 1007616 | consumed tokens: 2063597568 | elapsed time per iteration (s): 140.09 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.025222E+00 | grad norm: 0.467 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.619 | TFLOPs: 149.24 | -[default7]: iteration 493/ 3100 | consumed samples: 1009664 | consumed tokens: 2067791872 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.055796E+00 | grad norm: 0.540 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 494/ 3100 | consumed samples: 1011712 | consumed tokens: 2071986176 | elapsed time per iteration (s): 141.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.032478E+00 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 495/ 3100 | consumed samples: 1013760 | consumed tokens: 2076180480 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.045664E+00 | grad norm: 0.450 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 496/ 3100 | consumed samples: 1015808 | consumed tokens: 2080374784 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.051367E+00 | grad norm: 0.500 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.84 | -[default7]: iteration 497/ 3100 | consumed samples: 1017856 | consumed tokens: 2084569088 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.042184E+00 | grad norm: 0.460 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 498/ 3100 | consumed samples: 1019904 | consumed tokens: 2088763392 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.051828E+00 | grad norm: 0.451 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default0]:saving checkpoint at iteration 498 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-08 17:43:19,224] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step498 is begin to save! -[default4]:[2022-09-08 17:43:19,248] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_33-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,239] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_58-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,248] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_32-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,239] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_59-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,396] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_68-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_20-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_39-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_06-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,371] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_04-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_22-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_53-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_52-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,415] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_19-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_29-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_11-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_51-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,420] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_01-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_43-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,371] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_05-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_45-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_48-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,411] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_71_model_states.pt... -[default4]:[2022-09-08 17:43:19,426] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_71_model_states.pt. -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_12-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,416] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_24-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_28-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_41-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_70-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,396] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_69-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_08-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_26-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_49-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_42-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,416] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_25-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_31-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_30-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_14-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_17-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_09-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_16-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_38-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_54-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_21-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_23-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_57-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_15-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_71-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_27-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_40-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,457] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_67-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_56-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_50-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,455] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_66-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_65-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_61-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_07-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,415] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_18-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_03-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_64-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_60-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,455] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_37-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,419] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_34-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_46-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_72-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_10-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,419] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_35-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_63-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_47-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_55-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_44-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_62-model_00-model_states.pt... -[default4]:[2022-09-08 17:43:19,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_13-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:19,455] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_36-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:22,511] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_32-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,511] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_30_model_states.pt... -[default0]:[2022-09-08 17:43:22,518] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_30_model_states.pt. -[default4]:[2022-09-08 17:43:22,649] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_05-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:22,650] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_03_model_states.pt... -[default0]:[2022-09-08 17:43:22,667] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_58-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,667] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_56_model_states.pt... -[default4]:[2022-09-08 17:43:22,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_09-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:22,670] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_07_model_states.pt... -[default4]:[2022-09-08 17:43:22,673] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_07_model_states.pt. -[default0]:[2022-09-08 17:43:22,710] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_72-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,710] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_74-model_00-model_states.pt... -[default0]:[2022-09-08 17:43:22,712] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_74-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,712] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_70_model_states.pt... -[default4]:[2022-09-08 17:43:22,655] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_03_model_states.pt. -[default4]:[2022-09-08 17:43:22,754] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_33-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:22,755] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_31_model_states.pt... -[default4]:[2022-09-08 17:43:22,756] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_31_model_states.pt. -[default0]:[2022-09-08 17:43:22,679] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_56_model_states.pt. -[default0]:[2022-09-08 17:43:22,733] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_14-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,734] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_12_model_states.pt... -[default0]:[2022-09-08 17:43:22,738] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_12_model_states.pt. -[default0]:[2022-09-08 17:43:22,713] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_70_model_states.pt. -[default0]:[2022-09-08 17:43:22,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_44-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,797] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_42_model_states.pt... -[default0]:[2022-09-08 17:43:22,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_42_model_states.pt. -[default0]:[2022-09-08 17:43:22,819] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_06-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,819] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_04_model_states.pt... -[default0]:[2022-09-08 17:43:22,824] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_04_model_states.pt. -[default0]:[2022-09-08 17:43:22,763] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_28-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,764] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_26_model_states.pt... -[default0]:[2022-09-08 17:43:22,766] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_26_model_states.pt. -[default4]:[2022-09-08 17:43:22,863] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_17-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:22,864] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_15_model_states.pt... -[default4]:[2022-09-08 17:43:22,867] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_15_model_states.pt. -[default4]:[2022-09-08 17:43:22,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_15-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:22,866] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_13_model_states.pt... -[default4]:[2022-09-08 17:43:22,867] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_13_model_states.pt. -[default0]:[2022-09-08 17:43:22,825] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_10-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,825] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_08_model_states.pt... -[default0]:[2022-09-08 17:43:22,828] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_08_model_states.pt. -[default4]:[2022-09-08 17:43:22,869] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_59-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:22,869] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_57_model_states.pt... -[default4]:[2022-09-08 17:43:22,871] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_57_model_states.pt. -[default0]:[2022-09-08 17:43:22,940] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_08-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,940] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_06_model_states.pt... -[default0]:[2022-09-08 17:43:22,942] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_06_model_states.pt. -[default4]:[2022-09-08 17:43:22,942] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_31-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:22,943] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_29_model_states.pt... -[default4]:[2022-09-08 17:43:22,945] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_29_model_states.pt. -[default4]:[2022-09-08 17:43:22,990] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_71-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:22,991] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_69_model_states.pt... -[default4]:[2022-09-08 17:43:22,990] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_27-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:22,991] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_25_model_states.pt... -[default4]:[2022-09-08 17:43:22,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_25_model_states.pt. -[default4]:[2022-09-08 17:43:22,945] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_13-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:22,946] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_11_model_states.pt... -[default4]:[2022-09-08 17:43:22,949] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_11_model_states.pt. -[default0]:[2022-09-08 17:43:22,963] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_24-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,964] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_22_model_states.pt... -[default0]:[2022-09-08 17:43:22,967] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_22_model_states.pt. -[default0]:[2022-09-08 17:43:23,067] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_30-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,068] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_28_model_states.pt... -[default0]:[2022-09-08 17:43:23,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_28_model_states.pt. -[default0]:[2022-09-08 17:43:22,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_16-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:22,994] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_14_model_states.pt... -[default0]:[2022-09-08 17:43:22,996] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_14_model_states.pt. -[default4]:[2022-09-08 17:43:23,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_21-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,005] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_19_model_states.pt... -[default4]:[2022-09-08 17:43:23,010] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_19_model_states.pt. -[default4]:[2022-09-08 17:43:22,999] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_69_model_states.pt. -[default0]:[2022-09-08 17:43:23,066] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_56-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,066] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_54_model_states.pt... -[default0]:[2022-09-08 17:43:23,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_54_model_states.pt. -[default0]:[2022-09-08 17:43:23,039] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_50-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,040] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_48_model_states.pt... -[default0]:[2022-09-08 17:43:23,045] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_48_model_states.pt. -[default4]:[2022-09-08 17:43:23,103] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_07-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,103] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_05_model_states.pt... -[default4]:[2022-09-08 17:43:23,105] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_05_model_states.pt. -[default0]:[2022-09-08 17:43:23,085] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_64-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,085] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_62_model_states.pt... -[default0]:[2022-09-08 17:43:23,102] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_62_model_states.pt. -[default0]:[2022-09-08 17:43:23,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_20-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_18_model_states.pt... -[default0]:[2022-09-08 17:43:23,048] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_18_model_states.pt. -[default0]:[2022-09-08 17:43:23,047] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_04-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,048] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_02_model_states.pt... -[default0]:[2022-09-08 17:43:23,049] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_02_model_states.pt. -[default4]:[2022-09-08 17:43:23,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_29-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,090] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_27_model_states.pt... -[default4]:[2022-09-08 17:43:23,092] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_27_model_states.pt. -[default4]:[2022-09-08 17:43:23,129] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_11-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,130] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_09_model_states.pt... -[default4]:[2022-09-08 17:43:23,131] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_09_model_states.pt. -[default4]:[2022-09-08 17:43:23,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_45-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,071] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_43_model_states.pt... -[default4]:[2022-09-08 17:43:23,088] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_43_model_states.pt. -[default0]:[2022-09-08 17:43:23,111] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_70-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,112] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_68_model_states.pt... -[default0]:[2022-09-08 17:43:23,114] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_68_model_states.pt. -[default0]:[2022-09-08 17:43:23,105] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_26-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,105] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_24_model_states.pt... -[default0]:[2022-09-08 17:43:23,107] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_24_model_states.pt. -[default4]:[2022-09-08 17:43:23,073] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_49-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,073] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_47_model_states.pt... -[default4]:[2022-09-08 17:43:23,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_47_model_states.pt. -[default4]:[2022-09-08 17:43:23,123] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_25-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,123] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_23_model_states.pt... -[default4]:[2022-09-08 17:43:23,125] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_23_model_states.pt. -[default4]:[2022-09-08 17:43:23,171] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_65-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,171] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_63_model_states.pt... -[default4]:[2022-09-08 17:43:23,173] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_63_model_states.pt. -[default4]:[2022-09-08 17:43:23,193] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_63-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,193] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_61_model_states.pt... -[default4]:[2022-09-08 17:43:23,196] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_61_model_states.pt. -[default4]:[2022-09-08 17:43:23,198] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_39-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,198] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_37_model_states.pt... -[default4]:[2022-09-08 17:43:23,201] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_37_model_states.pt. -[default0]:[2022-09-08 17:43:23,154] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_36-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,155] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_34_model_states.pt... -[default0]:[2022-09-08 17:43:23,164] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_34_model_states.pt. -[default4]:[2022-09-08 17:43:23,208] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_51-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,208] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_49_model_states.pt... -[default4]:[2022-09-08 17:43:23,210] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_49_model_states.pt. -[default0]:[2022-09-08 17:43:23,237] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_48-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,237] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_46_model_states.pt... -[default0]:[2022-09-08 17:43:23,239] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_46_model_states.pt. -[default0]:[2022-09-08 17:43:23,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_12-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,221] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_10_model_states.pt... -[default0]:[2022-09-08 17:43:23,223] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_10_model_states.pt. -[default0]:[2022-09-08 17:43:23,265] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_38-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,265] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_36_model_states.pt... -[default0]:[2022-09-08 17:43:23,272] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_36_model_states.pt. -[default4]:[2022-09-08 17:43:23,237] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_23-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,237] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_21_model_states.pt... -[default4]:[2022-09-08 17:43:23,240] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_21_model_states.pt. -[default4]:[2022-09-08 17:43:23,200] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_57-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,200] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_55_model_states.pt... -[default4]:[2022-09-08 17:43:23,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_55_model_states.pt. -[default0]:[2022-09-08 17:43:23,260] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_60-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,260] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_58_model_states.pt... -[default0]:[2022-09-08 17:43:23,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_58_model_states.pt. -[default4]:[2022-09-08 17:43:23,296] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_37-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,297] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_35_model_states.pt... -[default4]:[2022-09-08 17:43:23,299] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_35_model_states.pt. -[default0]:[2022-09-08 17:43:23,225] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_46-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,225] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_44_model_states.pt... -[default0]:[2022-09-08 17:43:23,228] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_44_model_states.pt. -[default4]:[2022-09-08 17:43:23,263] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_35-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,263] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_33_model_states.pt... -[default4]:[2022-09-08 17:43:23,265] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_33_model_states.pt. -[default4]:[2022-09-08 17:43:23,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_55-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,305] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_53_model_states.pt... -[default4]:[2022-09-08 17:43:23,308] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_53_model_states.pt. -[default0]:[2022-09-08 17:43:23,303] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_62-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,304] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_60_model_states.pt... -[default0]:[2022-09-08 17:43:23,306] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_60_model_states.pt. -[default0]:[2022-09-08 17:43:23,322] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_22-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,323] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_20_model_states.pt... -[default0]:[2022-09-08 17:43:23,326] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_20_model_states.pt. -[default4]:[2022-09-08 17:43:23,325] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_19-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,325] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_17_model_states.pt... -[default4]:[2022-09-08 17:43:23,327] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_17_model_states.pt. -[default4]:[2022-09-08 17:43:23,288] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_43-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,288] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_41_model_states.pt... -[default4]:[2022-09-08 17:43:23,292] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_41_model_states.pt. -[default4]:[2022-09-08 17:43:23,267] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_41-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,268] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_39_model_states.pt... -[default4]:[2022-09-08 17:43:23,270] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_39_model_states.pt. -[default4]:[2022-09-08 17:43:23,360] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_69-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,360] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_67_model_states.pt... -[default4]:[2022-09-08 17:43:23,362] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_67_model_states.pt. -[default0]:[2022-09-08 17:43:23,297] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_42-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,297] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_40_model_states.pt... -[default0]:[2022-09-08 17:43:23,299] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_40_model_states.pt. -[default0]:[2022-09-08 17:43:23,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_54-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,336] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_52_model_states.pt... -[default0]:[2022-09-08 17:43:23,337] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_52_model_states.pt. -[default0]:[2022-09-08 17:43:23,348] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_40-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_38_model_states.pt... -[default0]:[2022-09-08 17:43:23,350] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_38_model_states.pt. -[default4]:[2022-09-08 17:43:23,349] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_61-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,349] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_59_model_states.pt... -[default4]:[2022-09-08 17:43:23,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_59_model_states.pt. -[default0]:[2022-09-08 17:43:23,334] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_68-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,334] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_66_model_states.pt... -[default0]:[2022-09-08 17:43:23,340] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_66_model_states.pt. -[default0]:[2022-09-08 17:43:23,353] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_18-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,353] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_16_model_states.pt... -[default0]:[2022-09-08 17:43:23,355] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_16_model_states.pt. -[default0]:[2022-09-08 17:43:23,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_34-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,311] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_32_model_states.pt... -[default0]:[2022-09-08 17:43:23,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_32_model_states.pt. -[default4]:[2022-09-08 17:43:23,359] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_47-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,359] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_45_model_states.pt... -[default4]:[2022-09-08 17:43:23,361] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_45_model_states.pt. -[default4]:[2022-09-08 17:43:23,531] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_67-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,531] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_65_model_states.pt... -[default4]:[2022-09-08 17:43:23,537] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_65_model_states.pt. -[default0]:[2022-09-08 17:43:23,552] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_66-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,552] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_64_model_states.pt... -[default0]:[2022-09-08 17:43:23,555] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_64_model_states.pt. -[default4]:[2022-09-08 17:43:23,542] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_53-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,543] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_51_model_states.pt... -[default4]:[2022-09-08 17:43:23,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_51_model_states.pt. -[default0]:[2022-09-08 17:43:23,583] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_52-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:23,583] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_50_model_states.pt... -[default0]:[2022-09-08 17:43:23,586] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_50_model_states.pt. -[default4]:[2022-09-08 17:43:23,721] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_03-model_00-model_states.pt. -[default4]:[2022-09-08 17:43:23,721] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_01_model_states.pt... -[default4]:[2022-09-08 17:43:23,726] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_01_model_states.pt. -[default0]:[2022-09-08 17:43:25,098] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/layer_01-model_00-model_states.pt. -[default0]:[2022-09-08 17:43:25,099] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_00_model_states.pt -[default0]:[2022-09-08 17:43:25,099] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_00_model_states.pt... -[default0]:[2022-09-08 17:43:25,101] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/mp_rank_00_model_states.pt. -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default5]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default3]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default7]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default1]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default0]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default4]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default6]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default2]:[2022-09-08 17:43:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default3]:[2022-09-08 17:43:34,505] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-08 17:43:34,505] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default3]:[2022-09-08 17:43:34,730] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-08 17:43:34,731] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default3]:[2022-09-08 17:43:35,074] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-08 17:43:35,075] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default2]:[2022-09-08 17:43:35,169] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-08 17:43:35,170] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default1]:[2022-09-08 17:43:35,166] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-08 17:43:35,167] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default4]:[2022-09-08 17:43:35,297] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-08 17:43:35,297] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default2]:[2022-09-08 17:43:35,315] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-08 17:43:35,316] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default0]:[2022-09-08 17:43:35,354] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-08 17:43:35,355] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default6]:[2022-09-08 17:43:35,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-08 17:43:35,455] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default7]:[2022-09-08 17:43:35,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-08 17:43:35,417] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default6]:[2022-09-08 17:43:35,495] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-08 17:43:35,495] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default6]:[2022-09-08 17:43:35,710] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-08 17:43:35,710] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default0]:[2022-09-08 17:43:35,879] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-08 17:43:35,879] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default5]:[2022-09-08 17:43:36,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-08 17:43:36,156] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default4]:[2022-09-08 17:43:36,329] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-08 17:43:36,329] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default0]:[2022-09-08 17:43:36,405] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-08 17:43:36,405] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default2]:[2022-09-08 17:43:36,715] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-08 17:43:36,715] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default0]:[2022-09-08 17:43:36,677] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-08 17:43:36,677] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default5]:[2022-09-08 17:43:36,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-08 17:43:36,770] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default0]:[2022-09-08 17:43:36,705] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-08 17:43:36,705] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default1]:[2022-09-08 17:43:36,832] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-08 17:43:36,833] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default2]:[2022-09-08 17:43:36,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-08 17:43:36,993] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default2]:[2022-09-08 17:43:37,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-08 17:43:37,036] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default4]:[2022-09-08 17:43:37,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-08 17:43:37,070] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default2]:[2022-09-08 17:43:37,075] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-08 17:43:37,075] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default1]:[2022-09-08 17:43:37,155] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-08 17:43:37,155] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default7]:[2022-09-08 17:43:37,241] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-08 17:43:37,241] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default4]:[2022-09-08 17:43:37,212] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-08 17:43:37,212] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default1]:[2022-09-08 17:43:37,303] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-08 17:43:37,303] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default6]:[2022-09-08 17:43:37,267] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-08 17:43:37,267] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default2]:[2022-09-08 17:43:37,330] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-08 17:43:37,331] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default0]:[2022-09-08 17:43:37,406] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-08 17:43:37,406] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default6]:[2022-09-08 17:43:37,419] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-08 17:43:37,419] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default6]:[2022-09-08 17:43:37,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-08 17:43:37,455] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default5]:[2022-09-08 17:43:37,461] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-08 17:43:37,461] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default6]:[2022-09-08 17:43:37,519] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-08 17:43:37,520] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default0]:[2022-09-08 17:43:37,565] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-08 17:43:37,566] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default3]:[2022-09-08 17:43:37,622] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-08 17:43:37,622] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default0]:[2022-09-08 17:43:37,631] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-08 17:43:37,631] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default1]:[2022-09-08 17:43:37,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-08 17:43:37,700] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default3]:[2022-09-08 17:43:37,696] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-08 17:43:37,696] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default7]:[2022-09-08 17:43:37,710] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-08 17:43:37,711] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default6]:[2022-09-08 17:43:37,686] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-08 17:43:37,687] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default3]:[2022-09-08 17:43:37,687] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-08 17:43:37,687] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default0]:[2022-09-08 17:43:37,747] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-08 17:43:37,747] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default7]:[2022-09-08 17:43:37,791] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-08 17:43:37,791] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default7]:[2022-09-08 17:43:37,803] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-08 17:43:37,803] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default4]:[2022-09-08 17:43:37,849] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-08 17:43:37,849] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default5]:[2022-09-08 17:43:37,824] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-08 17:43:37,824] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default4]:[2022-09-08 17:43:37,899] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-08 17:43:37,899] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default2]:[2022-09-08 17:43:37,920] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-08 17:43:37,920] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default7]:[2022-09-08 17:43:37,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-08 17:43:37,975] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default1]:[2022-09-08 17:43:38,002] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-08 17:43:38,002] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default7]:[2022-09-08 17:43:38,003] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-08 17:43:38,003] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default4]:[2022-09-08 17:43:37,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-08 17:43:37,980] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default1]:[2022-09-08 17:43:38,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-08 17:43:38,036] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default1]:[2022-09-08 17:43:38,094] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-08 17:43:38,095] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default3]:[2022-09-08 17:43:38,171] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-08 17:43:38,171] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default5]:[2022-09-08 17:43:38,239] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-08 17:43:38,239] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default2]:[2022-09-08 17:43:38,292] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-08 17:43:38,292] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default6]:[2022-09-08 17:43:38,214] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-08 17:43:38,214] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default1]:[2022-09-08 17:43:38,239] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-08 17:43:38,239] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default7]:[2022-09-08 17:43:38,266] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-08 17:43:38,266] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default1]:[2022-09-08 17:43:38,239] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-08 17:43:38,239] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default0]:[2022-09-08 17:43:38,239] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-08 17:43:38,239] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default5]:[2022-09-08 17:43:38,337] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-08 17:43:38,338] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default7]:[2022-09-08 17:43:38,337] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-08 17:43:38,337] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default3]:[2022-09-08 17:43:38,406] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-08 17:43:38,406] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default4]:[2022-09-08 17:43:38,376] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-08 17:43:38,376] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default7]:[2022-09-08 17:43:38,378] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-08 17:43:38,378] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default0]:[2022-09-08 17:43:38,376] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-08 17:43:38,376] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default4]:[2022-09-08 17:43:38,366] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-08 17:43:38,366] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default7]:[2022-09-08 17:43:38,368] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-08 17:43:38,368] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default0]:[2022-09-08 17:43:38,452] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-08 17:43:38,452] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default5]:[2022-09-08 17:43:38,394] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-08 17:43:38,394] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default2]:[2022-09-08 17:43:38,413] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-08 17:43:38,413] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default5]:[2022-09-08 17:43:38,529] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-08 17:43:38,529] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default2]:[2022-09-08 17:43:38,501] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-08 17:43:38,501] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default3]:[2022-09-08 17:43:38,515] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-08 17:43:38,515] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default2]:[2022-09-08 17:43:38,510] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-08 17:43:38,510] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default6]:[2022-09-08 17:43:38,556] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-08 17:43:38,557] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default7]:[2022-09-08 17:43:38,510] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-08 17:43:38,510] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default5]:[2022-09-08 17:43:38,620] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-08 17:43:38,620] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default4]:[2022-09-08 17:43:38,638] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-08 17:43:38,638] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default1]:[2022-09-08 17:43:38,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-08 17:43:38,637] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default1]:[2022-09-08 17:43:38,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-08 17:43:38,579] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default2]:[2022-09-08 17:43:38,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-08 17:43:38,648] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default5]:[2022-09-08 17:43:38,654] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-08 17:43:38,655] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default1]:[2022-09-08 17:43:38,626] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-08 17:43:38,626] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default4]:[2022-09-08 17:43:38,715] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-08 17:43:38,715] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default4]:[2022-09-08 17:43:38,661] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-08 17:43:38,661] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default2]:[2022-09-08 17:43:38,689] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-08 17:43:38,689] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default3]:[2022-09-08 17:43:38,678] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-08 17:43:38,678] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default0]:[2022-09-08 17:43:38,768] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-08 17:43:38,769] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default0]:[2022-09-08 17:43:38,688] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-08 17:43:38,688] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default3]:[2022-09-08 17:43:38,769] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-08 17:43:38,769] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default6]:[2022-09-08 17:43:38,801] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-08 17:43:38,801] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default2]:[2022-09-08 17:43:38,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-08 17:43:38,820] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default3]:[2022-09-08 17:43:38,816] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-08 17:43:38,816] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default6]:[2022-09-08 17:43:38,781] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-08 17:43:38,781] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default7]:[2022-09-08 17:43:38,799] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-08 17:43:38,799] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default1]:[2022-09-08 17:43:38,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-08 17:43:38,800] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default1]:[2022-09-08 17:43:38,796] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-08 17:43:38,796] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default7]:[2022-09-08 17:43:38,866] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-08 17:43:38,866] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default2]:[2022-09-08 17:43:38,806] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-08 17:43:38,806] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default1]:[2022-09-08 17:43:38,908] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-08 17:43:38,908] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default1]:[2022-09-08 17:43:38,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-08 17:43:38,932] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default5]:[2022-09-08 17:43:38,913] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-08 17:43:38,913] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default6]:[2022-09-08 17:43:38,880] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-08 17:43:38,880] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default5]:[2022-09-08 17:43:38,846] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-08 17:43:38,846] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default1]:[2022-09-08 17:43:38,855] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-08 17:43:38,855] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default4]:[2022-09-08 17:43:39,006] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-08 17:43:39,006] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default5]:[2022-09-08 17:43:38,936] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-08 17:43:38,936] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default0]:[2022-09-08 17:43:38,961] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-08 17:43:38,961] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default0]:[2022-09-08 17:43:38,934] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-08 17:43:38,934] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default2]:[2022-09-08 17:43:38,969] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-08 17:43:38,969] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default3]:[2022-09-08 17:43:38,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-08 17:43:38,994] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default4]:[2022-09-08 17:43:38,998] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-08 17:43:38,999] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default5]:[2022-09-08 17:43:39,049] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-08 17:43:39,049] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default2]:[2022-09-08 17:43:39,058] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-08 17:43:39,058] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default0]:[2022-09-08 17:43:39,100] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-08 17:43:39,100] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default3]:[2022-09-08 17:43:39,087] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-08 17:43:39,087] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default0]:[2022-09-08 17:43:39,085] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-08 17:43:39,085] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default3]:[2022-09-08 17:43:39,107] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-08 17:43:39,107] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default1]:[2022-09-08 17:43:39,158] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-08 17:43:39,158] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default5]:[2022-09-08 17:43:39,207] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-08 17:43:39,207] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default2]:[2022-09-08 17:43:39,246] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-08 17:43:39,246] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default6]:[2022-09-08 17:43:39,340] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-08 17:43:39,340] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default3]:[2022-09-08 17:43:39,317] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-08 17:43:39,317] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default6]:[2022-09-08 17:43:39,269] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-08 17:43:39,270] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default3]:[2022-09-08 17:43:39,300] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-08 17:43:39,300] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default6]:[2022-09-08 17:43:39,356] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-08 17:43:39,356] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default4]:[2022-09-08 17:43:39,344] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-08 17:43:39,344] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default4]:[2022-09-08 17:43:39,341] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-08 17:43:39,342] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default0]:[2022-09-08 17:43:39,337] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-08 17:43:39,337] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default7]:[2022-09-08 17:43:39,352] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-08 17:43:39,352] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default7]:[2022-09-08 17:43:39,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-08 17:43:39,407] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default4]:[2022-09-08 17:43:39,447] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-08 17:43:39,447] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default7]:[2022-09-08 17:43:39,410] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-08 17:43:39,410] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default7]:[2022-09-08 17:43:39,418] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-08 17:43:39,418] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default3]:[2022-09-08 17:43:39,504] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-08 17:43:39,504] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default6]:[2022-09-08 17:43:39,495] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-08 17:43:39,495] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default6]:[2022-09-08 17:43:39,467] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-08 17:43:39,467] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default4]:[2022-09-08 17:43:39,525] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-08 17:43:39,525] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default5]:[2022-09-08 17:43:39,583] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-08 17:43:39,584] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default5]:[2022-09-08 17:43:39,568] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-08 17:43:39,568] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default3]:[2022-09-08 17:43:39,616] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-08 17:43:39,616] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default1]:[2022-09-08 17:43:39,718] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-08 17:43:39,718] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default2]:[2022-09-08 17:43:39,648] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-08 17:43:39,648] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default7]:[2022-09-08 17:43:39,717] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-08 17:43:39,717] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default4]:[2022-09-08 17:43:39,792] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-08 17:43:39,793] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default1]:[2022-09-08 17:43:39,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-08 17:43:39,808] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default5]:[2022-09-08 17:43:39,804] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-08 17:43:39,804] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default5]:[2022-09-08 17:43:39,812] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-08 17:43:39,812] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default3]:[2022-09-08 17:43:39,853] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-08 17:43:39,853] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default2]:[2022-09-08 17:43:39,769] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-08 17:43:39,769] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default1]:[2022-09-08 17:43:39,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-08 17:43:39,856] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default7]:[2022-09-08 17:43:39,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-08 17:43:39,821] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default6]:[2022-09-08 17:43:39,867] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-08 17:43:39,867] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default1]:[2022-09-08 17:43:39,847] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-08 17:43:39,847] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default6]:[2022-09-08 17:43:39,838] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-08 17:43:39,838] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default4]:[2022-09-08 17:43:39,791] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-08 17:43:39,791] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default3]:[2022-09-08 17:43:39,807] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-08 17:43:39,807] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default4]:[2022-09-08 17:43:39,857] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-08 17:43:39,857] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default0]:[2022-09-08 17:43:39,917] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-08 17:43:39,918] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default7]:[2022-09-08 17:43:39,911] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-08 17:43:39,911] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default7]:[2022-09-08 17:43:39,948] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-08 17:43:39,948] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default6]:[2022-09-08 17:43:39,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-08 17:43:39,910] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default5]:[2022-09-08 17:43:39,904] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-08 17:43:39,905] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default2]:[2022-09-08 17:43:39,880] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-08 17:43:39,880] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default4]:[2022-09-08 17:43:40,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-08 17:43:40,011] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default7]:[2022-09-08 17:43:39,926] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-08 17:43:39,927] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default0]:[2022-09-08 17:43:39,959] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-08 17:43:39,959] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default1]:[2022-09-08 17:43:39,956] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-08 17:43:39,956] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default0]:[2022-09-08 17:43:39,959] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-08 17:43:39,959] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default2]:[2022-09-08 17:43:40,103] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-08 17:43:40,103] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default0]:[2022-09-08 17:43:40,061] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-08 17:43:40,061] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default4]:[2022-09-08 17:43:40,083] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-08 17:43:40,083] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default6]:[2022-09-08 17:43:40,103] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-08 17:43:40,103] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default4]:[2022-09-08 17:43:40,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-08 17:43:40,077] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default3]:[2022-09-08 17:43:40,163] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-08 17:43:40,163] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default5]:[2022-09-08 17:43:40,127] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-08 17:43:40,127] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default6]:[2022-09-08 17:43:40,194] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-08 17:43:40,194] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default5]:[2022-09-08 17:43:40,185] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-08 17:43:40,185] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default1]:[2022-09-08 17:43:40,211] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-08 17:43:40,211] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default4]:[2022-09-08 17:43:40,228] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-08 17:43:40,228] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default3]:[2022-09-08 17:43:40,167] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-08 17:43:40,167] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default0]:[2022-09-08 17:43:40,195] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-08 17:43:40,195] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default2]:[2022-09-08 17:43:40,182] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-08 17:43:40,182] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default5]:[2022-09-08 17:43:40,265] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-08 17:43:40,266] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default0]:[2022-09-08 17:43:40,202] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-08 17:43:40,202] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default3]:[2022-09-08 17:43:40,242] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-08 17:43:40,243] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default7]:[2022-09-08 17:43:40,336] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-08 17:43:40,336] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default2]:[2022-09-08 17:43:40,355] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-08 17:43:40,355] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default4]:[2022-09-08 17:43:40,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-08 17:43:40,305] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default4]:[2022-09-08 17:43:40,349] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-08 17:43:40,350] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default3]:[2022-09-08 17:43:40,363] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-08 17:43:40,363] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default7]:[2022-09-08 17:43:40,381] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-08 17:43:40,381] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default1]:[2022-09-08 17:43:40,394] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-08 17:43:40,394] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default5]:[2022-09-08 17:43:40,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-08 17:43:40,416] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default0]:[2022-09-08 17:43:40,465] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-08 17:43:40,465] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default6]:[2022-09-08 17:43:40,482] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-08 17:43:40,482] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default6]:[2022-09-08 17:43:40,469] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-08 17:43:40,469] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default0]:[2022-09-08 17:43:40,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-08 17:43:40,416] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default0]:[2022-09-08 17:43:40,469] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-08 17:43:40,469] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default6]:[2022-09-08 17:43:40,477] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-08 17:43:40,477] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default7]:[2022-09-08 17:43:40,501] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-08 17:43:40,501] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default3]:[2022-09-08 17:43:40,521] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-08 17:43:40,521] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default6]:[2022-09-08 17:43:40,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-08 17:43:40,488] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default2]:[2022-09-08 17:43:40,562] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-08 17:43:40,562] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default3]:[2022-09-08 17:43:40,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-08 17:43:40,580] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default5]:[2022-09-08 17:43:40,563] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-08 17:43:40,563] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default2]:[2022-09-08 17:43:40,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-08 17:43:40,595] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default4]:[2022-09-08 17:43:40,656] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-08 17:43:40,656] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default0]:[2022-09-08 17:43:40,719] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-08 17:43:40,719] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default7]:[2022-09-08 17:43:40,742] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-08 17:43:40,742] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default2]:[2022-09-08 17:43:40,729] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-08 17:43:40,729] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default5]:[2022-09-08 17:43:40,747] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-08 17:43:40,747] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default2]:[2022-09-08 17:43:40,759] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-08 17:43:40,759] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default3]:[2022-09-08 17:43:40,847] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-08 17:43:40,847] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default1]:[2022-09-08 17:43:40,829] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-08 17:43:40,830] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default3]:[2022-09-08 17:43:40,929] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-08 17:43:40,929] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default0]:[2022-09-08 17:43:40,923] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-08 17:43:40,923] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default1]:[2022-09-08 17:43:40,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-08 17:43:40,910] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default1]:[2022-09-08 17:43:40,995] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-08 17:43:40,995] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default2]:[2022-09-08 17:43:41,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-08 17:43:41,098] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default6]:[2022-09-08 17:43:41,092] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-08 17:43:41,092] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default4]:[2022-09-08 17:43:41,127] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-08 17:43:41,127] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default6]:[2022-09-08 17:43:41,155] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-08 17:43:41,156] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default5]:[2022-09-08 17:43:41,153] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-08 17:43:41,153] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default1]:[2022-09-08 17:43:41,309] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-08 17:43:41,310] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default7]:[2022-09-08 17:43:41,344] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-08 17:43:41,345] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default4]:[2022-09-08 17:43:41,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-08 17:43:41,351] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default7]:[2022-09-08 17:43:41,431] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-08 17:43:41,431] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default4]:[2022-09-08 17:43:41,390] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-08 17:43:41,390] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default7]:[2022-09-08 17:43:41,469] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-08 17:43:41,469] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default7]:[2022-09-08 17:43:41,476] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-08 17:43:41,476] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default3]:[2022-09-08 17:43:41,418] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-08 17:43:41,418] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default3]:[2022-09-08 17:43:41,422] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-08 17:43:41,422] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default2]:[2022-09-08 17:43:41,532] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-08 17:43:41,532] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default5]:[2022-09-08 17:43:41,543] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-08 17:43:41,544] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default5]:[2022-09-08 17:43:41,592] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-08 17:43:41,592] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default3]:[2022-09-08 17:43:41,634] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-08 17:43:41,634] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default0]:[2022-09-08 17:43:41,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-08 17:43:41,676] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default1]:[2022-09-08 17:43:41,679] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-08 17:43:41,679] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default5]:[2022-09-08 17:43:41,765] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-08 17:43:41,765] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default5]:[2022-09-08 17:43:41,817] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-08 17:43:41,818] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default6]:[2022-09-08 17:43:41,845] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-08 17:43:41,846] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default4]:[2022-09-08 17:43:41,940] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-08 17:43:41,940] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default5]:[2022-09-08 17:43:41,977] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-08 17:43:41,977] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default0]:[2022-09-08 17:43:42,126] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-08 17:43:42,126] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default7]:[2022-09-08 17:43:42,122] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-08 17:43:42,122] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default1]:[2022-09-08 17:43:42,239] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-08 17:43:42,240] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default3]:[2022-09-08 17:43:42,230] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-08 17:43:42,230] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default7]:[2022-09-08 17:43:42,382] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-08 17:43:42,382] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default4]:[2022-09-08 17:43:42,367] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-08 17:43:42,368] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default6]:[2022-09-08 17:43:42,459] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-08 17:43:42,459] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default0]:[2022-09-08 17:43:42,439] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-08 17:43:42,440] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default2]:[2022-09-08 17:43:42,518] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-08 17:43:42,519] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default1]:[2022-09-08 17:43:42,593] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-08 17:43:42,593] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default5]:[2022-09-08 17:43:42,694] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-08 17:43:42,694] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default7]:[2022-09-08 17:43:42,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-08 17:43:42,784] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default6]:[2022-09-08 17:43:42,801] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-08 17:43:42,801] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default6]:[2022-09-08 17:43:42,869] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-08 17:43:42,869] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default4]:[2022-09-08 17:43:42,972] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-08 17:43:42,972] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default2]:[2022-09-08 17:43:43,261] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-08 17:43:43,262] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default5]:[2022-09-08 17:43:43,399] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-08 17:43:43,399] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default6]:[2022-09-08 17:43:43,957] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-08 17:43:43,957] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default0]:[2022-09-08 17:43:46,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-08 17:43:46,675] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default1]:[2022-09-08 17:43:46,762] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-08 17:43:46,763] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default1]:[2022-09-08 17:43:46,761] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-08 17:43:46,761] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default2]:[2022-09-08 17:43:46,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-08 17:43:46,823] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default6]:[2022-09-08 17:43:46,976] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-08 17:43:46,976] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default3]:[2022-09-08 17:43:47,062] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-08 17:43:47,063] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default2]:[2022-09-08 17:43:47,474] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-08 17:43:47,474] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default4]:[2022-09-08 17:43:47,510] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-08 17:43:47,510] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default5]:[2022-09-08 17:43:47,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-08 17:43:47,850] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default7]:[2022-09-08 17:43:48,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-08 17:43:48,051] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default3]:[2022-09-08 17:43:49,099] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-08 17:43:49,099] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default0]:[2022-09-08 17:43:49,393] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-08 17:43:49,393] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default3]:[2022-09-08 17:43:49,493] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-08 17:43:49,493] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default0]:[2022-09-08 17:43:50,752] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-08 17:43:50,754] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default1]:[2022-09-08 17:43:51,143] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-08 17:43:51,143] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default2]:[2022-09-08 17:43:52,345] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-08 17:43:52,345] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default6]:[2022-09-08 17:44:04,568] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-08 17:44:04,569] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default7]:[2022-09-08 17:44:04,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-08 17:44:04,576] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:time (ms) | save-checkpoint: 47975.68 -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,159] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-08 17:44:07,159] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]: successfully saved checkpoint at iteration 498 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,107] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-08 17:44:07,107] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step498/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default5]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default6]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default3]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default2]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default4]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default1]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default0]:[2022-09-08 17:44:07,160] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step498 is ready now! -[default7]: iteration 499/ 3100 | consumed samples: 1021952 | consumed tokens: 2092957696 | elapsed time per iteration (s): 189.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.049284E+00 | grad norm: 0.528 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 10.807 | TFLOPs: 110.32 | -[default7]: iteration 500/ 3100 | consumed samples: 1024000 | consumed tokens: 2097152000 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.029777E+00 | grad norm: 0.438 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]:---------------------------------------------------------------------------------------------------------- -[default7]:validation_pretraining loss at iteration 500 | lm loss value: 2.311688E+00 | lm loss PPL: 1.009145E+01 | -[default7]:---------------------------------------------------------------------------------------------------------- -[default7]: iteration 501/ 3100 | consumed samples: 1026048 | consumed tokens: 2101346304 | elapsed time per iteration (s): 188.26 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.039266E+00 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 10.879 | TFLOPs: 111.06 | -[default7]: iteration 502/ 3100 | consumed samples: 1028096 | consumed tokens: 2105540608 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.064137E+00 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 503/ 3100 | consumed samples: 1030144 | consumed tokens: 2109734912 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.066179E+00 | grad norm: 0.461 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 504/ 3100 | consumed samples: 1032192 | consumed tokens: 2113929216 | elapsed time per iteration (s): 140.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.049495E+00 | grad norm: 0.436 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.568 | TFLOPs: 148.72 | -[default7]: iteration 505/ 3100 | consumed samples: 1034240 | consumed tokens: 2118123520 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.043128E+00 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 506/ 3100 | consumed samples: 1036288 | consumed tokens: 2122317824 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.043437E+00 | grad norm: 0.468 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 507/ 3100 | consumed samples: 1038336 | consumed tokens: 2126512128 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.046598E+00 | grad norm: 0.438 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.430 | TFLOPs: 147.31 | -[default7]: iteration 508/ 3100 | consumed samples: 1040384 | consumed tokens: 2130706432 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.051975E+00 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 509/ 3100 | consumed samples: 1042432 | consumed tokens: 2134900736 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.054838E+00 | grad norm: 0.457 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 510/ 3100 | consumed samples: 1044480 | consumed tokens: 2139095040 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.049994E+00 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.47 | -[default7]: iteration 511/ 3100 | consumed samples: 1046528 | consumed tokens: 2143289344 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.047495E+00 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.37 | -[default7]: iteration 512/ 3100 | consumed samples: 1048576 | consumed tokens: 2147483648 | elapsed time per iteration (s): 140.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.059098E+00 | grad norm: 0.469 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.536 | TFLOPs: 148.39 | -[default7]: iteration 513/ 3100 | consumed samples: 1050624 | consumed tokens: 2151677952 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.046977E+00 | grad norm: 0.460 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 514/ 3100 | consumed samples: 1052672 | consumed tokens: 2155872256 | elapsed time per iteration (s): 140.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.036470E+00 | grad norm: 0.459 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.557 | TFLOPs: 148.61 | -[default7]: iteration 515/ 3100 | consumed samples: 1054720 | consumed tokens: 2160066560 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.058003E+00 | grad norm: 0.491 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 516/ 3100 | consumed samples: 1056768 | consumed tokens: 2164260864 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.041692E+00 | grad norm: 0.513 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 517/ 3100 | consumed samples: 1058816 | consumed tokens: 2168455168 | elapsed time per iteration (s): 140.07 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.043809E+00 | grad norm: 0.485 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.621 | TFLOPs: 149.26 | -[default7]: iteration 518/ 3100 | consumed samples: 1060864 | consumed tokens: 2172649472 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.035305E+00 | grad norm: 0.465 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 519/ 3100 | consumed samples: 1062912 | consumed tokens: 2176843776 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.038189E+00 | grad norm: 0.510 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 520/ 3100 | consumed samples: 1064960 | consumed tokens: 2181038080 | elapsed time per iteration (s): 140.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.032595E+00 | grad norm: 0.446 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.559 | TFLOPs: 148.62 | -[default7]: iteration 521/ 3100 | consumed samples: 1067008 | consumed tokens: 2185232384 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.029536E+00 | grad norm: 0.504 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 522/ 3100 | consumed samples: 1069056 | consumed tokens: 2189426688 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.042303E+00 | grad norm: 0.439 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.80 | -[default7]: iteration 523/ 3100 | consumed samples: 1071104 | consumed tokens: 2193620992 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.048731E+00 | grad norm: 0.482 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 524/ 3100 | consumed samples: 1073152 | consumed tokens: 2197815296 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.029517E+00 | grad norm: 0.455 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.80 | -[default7]: iteration 525/ 3100 | consumed samples: 1075200 | consumed tokens: 2202009600 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.026833E+00 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 526/ 3100 | consumed samples: 1077248 | consumed tokens: 2206203904 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.037425E+00 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 527/ 3100 | consumed samples: 1079296 | consumed tokens: 2210398208 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.044116E+00 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 528/ 3100 | consumed samples: 1081344 | consumed tokens: 2214592512 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.043270E+00 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 529/ 3100 | consumed samples: 1083392 | consumed tokens: 2218786816 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.038964E+00 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 530/ 3100 | consumed samples: 1085440 | consumed tokens: 2222981120 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.036500E+00 | grad norm: 0.423 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 531/ 3100 | consumed samples: 1087488 | consumed tokens: 2227175424 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.035592E+00 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 532/ 3100 | consumed samples: 1089536 | consumed tokens: 2231369728 | elapsed time per iteration (s): 140.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.049861E+00 | grad norm: 0.497 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.572 | TFLOPs: 148.76 | -[default7]: iteration 533/ 3100 | consumed samples: 1091584 | consumed tokens: 2235564032 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.032725E+00 | grad norm: 0.477 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 534/ 3100 | consumed samples: 1093632 | consumed tokens: 2239758336 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.036812E+00 | grad norm: 0.517 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 535/ 3100 | consumed samples: 1095680 | consumed tokens: 2243952640 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.040302E+00 | grad norm: 0.489 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 536/ 3100 | consumed samples: 1097728 | consumed tokens: 2248146944 | elapsed time per iteration (s): 142.17 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.027173E+00 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.405 | TFLOPs: 147.05 | -[default7]: iteration 537/ 3100 | consumed samples: 1099776 | consumed tokens: 2252341248 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.044056E+00 | grad norm: 0.446 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.44 | -[default7]: iteration 538/ 3100 | consumed samples: 1101824 | consumed tokens: 2256535552 | elapsed time per iteration (s): 140.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.034849E+00 | grad norm: 0.481 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.585 | TFLOPs: 148.89 | -[default7]: iteration 539/ 3100 | consumed samples: 1103872 | consumed tokens: 2260729856 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.035323E+00 | grad norm: 0.454 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 540/ 3100 | consumed samples: 1105920 | consumed tokens: 2264924160 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.016567E+00 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 541/ 3100 | consumed samples: 1107968 | consumed tokens: 2269118464 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.039212E+00 | grad norm: 0.477 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 542/ 3100 | consumed samples: 1110016 | consumed tokens: 2273312768 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.026637E+00 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 543/ 3100 | consumed samples: 1112064 | consumed tokens: 2277507072 | elapsed time per iteration (s): 141.04 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.010555E+00 | grad norm: 0.464 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.520 | TFLOPs: 148.23 | -[default7]: iteration 544/ 3100 | consumed samples: 1114112 | consumed tokens: 2281701376 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.038384E+00 | grad norm: 0.459 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.72 | -[default7]: iteration 545/ 3100 | consumed samples: 1116160 | consumed tokens: 2285895680 | elapsed time per iteration (s): 141.12 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.032516E+00 | grad norm: 0.477 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.513 | TFLOPs: 148.15 | -[default7]: iteration 546/ 3100 | consumed samples: 1118208 | consumed tokens: 2290089984 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.015527E+00 | grad norm: 0.436 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 547/ 3100 | consumed samples: 1120256 | consumed tokens: 2294284288 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.022175E+00 | grad norm: 0.518 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.47 | -[default7]: iteration 548/ 3100 | consumed samples: 1122304 | consumed tokens: 2298478592 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.026433E+00 | grad norm: 0.473 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 549/ 3100 | consumed samples: 1124352 | consumed tokens: 2302672896 | elapsed time per iteration (s): 140.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.021933E+00 | grad norm: 0.572 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.534 | TFLOPs: 148.37 | -[default7]: iteration 550/ 3100 | consumed samples: 1126400 | consumed tokens: 2306867200 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.042974E+00 | grad norm: 0.451 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 551/ 3100 | consumed samples: 1128448 | consumed tokens: 2311061504 | elapsed time per iteration (s): 139.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.018550E+00 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.648 | TFLOPs: 149.53 | -[default7]: iteration 552/ 3100 | consumed samples: 1130496 | consumed tokens: 2315255808 | elapsed time per iteration (s): 141.28 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.030188E+00 | grad norm: 0.477 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.496 | TFLOPs: 147.98 | -[default7]: iteration 553/ 3100 | consumed samples: 1132544 | consumed tokens: 2319450112 | elapsed time per iteration (s): 140.09 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.016342E+00 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.619 | TFLOPs: 149.24 | -[default7]: iteration 554/ 3100 | consumed samples: 1134592 | consumed tokens: 2323644416 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.021576E+00 | grad norm: 0.472 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 555/ 3100 | consumed samples: 1136640 | consumed tokens: 2327838720 | elapsed time per iteration (s): 141.22 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.035140E+00 | grad norm: 0.430 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.502 | TFLOPs: 148.05 | -[default7]: iteration 556/ 3100 | consumed samples: 1138688 | consumed tokens: 2332033024 | elapsed time per iteration (s): 140.30 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.031526E+00 | grad norm: 0.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.598 | TFLOPs: 149.02 | -[default7]: iteration 557/ 3100 | consumed samples: 1140736 | consumed tokens: 2336227328 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.024702E+00 | grad norm: 0.434 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 558/ 3100 | consumed samples: 1142784 | consumed tokens: 2340421632 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.041732E+00 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.62 | -[default7]: iteration 559/ 3100 | consumed samples: 1144832 | consumed tokens: 2344615936 | elapsed time per iteration (s): 140.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.031101E+00 | grad norm: 0.481 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.586 | TFLOPs: 148.90 | -[default7]: iteration 560/ 3100 | consumed samples: 1146880 | consumed tokens: 2348810240 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.021179E+00 | grad norm: 0.443 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 561/ 3100 | consumed samples: 1148928 | consumed tokens: 2353004544 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.018525E+00 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 562/ 3100 | consumed samples: 1150976 | consumed tokens: 2357198848 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.021499E+00 | grad norm: 0.500 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 563/ 3100 | consumed samples: 1153024 | consumed tokens: 2361393152 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.012324E+00 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.62 | -[default7]: iteration 564/ 3100 | consumed samples: 1155072 | consumed tokens: 2365587456 | elapsed time per iteration (s): 140.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.015278E+00 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.535 | TFLOPs: 148.38 | -[default7]: iteration 565/ 3100 | consumed samples: 1157120 | consumed tokens: 2369781760 | elapsed time per iteration (s): 139.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.031129E+00 | grad norm: 0.455 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.633 | TFLOPs: 149.38 | -[default7]: iteration 566/ 3100 | consumed samples: 1159168 | consumed tokens: 2373976064 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.006744E+00 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 567/ 3100 | consumed samples: 1161216 | consumed tokens: 2378170368 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.019041E+00 | grad norm: 0.449 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 568/ 3100 | consumed samples: 1163264 | consumed tokens: 2382364672 | elapsed time per iteration (s): 140.29 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.010888E+00 | grad norm: 0.495 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.598 | TFLOPs: 149.02 | -[default7]: iteration 569/ 3100 | consumed samples: 1165312 | consumed tokens: 2386558976 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.014929E+00 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 | -[default7]: iteration 570/ 3100 | consumed samples: 1167360 | consumed tokens: 2390753280 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.015115E+00 | grad norm: 0.514 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 571/ 3100 | consumed samples: 1169408 | consumed tokens: 2394947584 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.022312E+00 | grad norm: 0.447 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 572/ 3100 | consumed samples: 1171456 | consumed tokens: 2399141888 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.019532E+00 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.54 | -[default7]: iteration 573/ 3100 | consumed samples: 1173504 | consumed tokens: 2403336192 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.030953E+00 | grad norm: 0.465 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 574/ 3100 | consumed samples: 1175552 | consumed tokens: 2407530496 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.021040E+00 | grad norm: 0.447 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 575/ 3100 | consumed samples: 1177600 | consumed tokens: 2411724800 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.956906E-01 | grad norm: 0.513 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 576/ 3100 | consumed samples: 1179648 | consumed tokens: 2415919104 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.029125E+00 | grad norm: 0.468 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 577/ 3100 | consumed samples: 1181696 | consumed tokens: 2420113408 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.004534E+00 | grad norm: 0.423 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 578/ 3100 | consumed samples: 1183744 | consumed tokens: 2424307712 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.019833E+00 | grad norm: 0.464 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 579/ 3100 | consumed samples: 1185792 | consumed tokens: 2428502016 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.001671E+00 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 580/ 3100 | consumed samples: 1187840 | consumed tokens: 2432696320 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.009228E+00 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 581/ 3100 | consumed samples: 1189888 | consumed tokens: 2436890624 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.020631E+00 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 582/ 3100 | consumed samples: 1191936 | consumed tokens: 2441084928 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.003589E+00 | grad norm: 0.448 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 583/ 3100 | consumed samples: 1193984 | consumed tokens: 2445279232 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.010324E+00 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 584/ 3100 | consumed samples: 1196032 | consumed tokens: 2449473536 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.017357E+00 | grad norm: 0.430 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 585/ 3100 | consumed samples: 1198080 | consumed tokens: 2453667840 | elapsed time per iteration (s): 140.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.014275E+00 | grad norm: 0.621 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.590 | TFLOPs: 148.94 | -[default7]: iteration 586/ 3100 | consumed samples: 1200128 | consumed tokens: 2457862144 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.016153E+00 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 587/ 3100 | consumed samples: 1202176 | consumed tokens: 2462056448 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.009745E+00 | grad norm: 0.512 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 588/ 3100 | consumed samples: 1204224 | consumed tokens: 2466250752 | elapsed time per iteration (s): 139.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.012745E+00 | grad norm: 0.464 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.632 | TFLOPs: 149.37 | -[default7]: iteration 589/ 3100 | consumed samples: 1206272 | consumed tokens: 2470445056 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.006071E+00 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 590/ 3100 | consumed samples: 1208320 | consumed tokens: 2474639360 | elapsed time per iteration (s): 140.26 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.940271E-01 | grad norm: 0.560 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.601 | TFLOPs: 149.05 | -[default7]: iteration 591/ 3100 | consumed samples: 1210368 | consumed tokens: 2478833664 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.002611E+00 | grad norm: 0.485 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 592/ 3100 | consumed samples: 1212416 | consumed tokens: 2483027968 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.935203E-01 | grad norm: 0.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 593/ 3100 | consumed samples: 1214464 | consumed tokens: 2487222272 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.012614E+00 | grad norm: 0.461 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 594/ 3100 | consumed samples: 1216512 | consumed tokens: 2491416576 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.013605E+00 | grad norm: 0.481 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 595/ 3100 | consumed samples: 1218560 | consumed tokens: 2495610880 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.000776E+00 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 596/ 3100 | consumed samples: 1220608 | consumed tokens: 2499805184 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.016243E+00 | grad norm: 0.499 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 597/ 3100 | consumed samples: 1222656 | consumed tokens: 2503999488 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.019975E+00 | grad norm: 0.457 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 598/ 3100 | consumed samples: 1224704 | consumed tokens: 2508193792 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.997981E-01 | grad norm: 0.449 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 599/ 3100 | consumed samples: 1226752 | consumed tokens: 2512388096 | elapsed time per iteration (s): 141.23 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.005990E+00 | grad norm: 0.471 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.501 | TFLOPs: 148.03 | -[default7]: iteration 600/ 3100 | consumed samples: 1228800 | consumed tokens: 2516582400 | elapsed time per iteration (s): 139.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.004096E+00 | grad norm: 0.482 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.641 | TFLOPs: 149.46 | -[default7]: iteration 601/ 3100 | consumed samples: 1230848 | consumed tokens: 2520776704 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.008503E+00 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 602/ 3100 | consumed samples: 1232896 | consumed tokens: 2524971008 | elapsed time per iteration (s): 140.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.937931E-01 | grad norm: 0.523 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.568 | TFLOPs: 148.72 | -[default7]: iteration 603/ 3100 | consumed samples: 1234944 | consumed tokens: 2529165312 | elapsed time per iteration (s): 141.05 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.939179E-01 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.520 | TFLOPs: 148.22 | -[default7]: iteration 604/ 3100 | consumed samples: 1236992 | consumed tokens: 2533359616 | elapsed time per iteration (s): 141.29 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.000902E+00 | grad norm: 0.436 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.495 | TFLOPs: 147.97 | -[default7]: iteration 605/ 3100 | consumed samples: 1239040 | consumed tokens: 2537553920 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.959083E-01 | grad norm: 0.481 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 606/ 3100 | consumed samples: 1241088 | consumed tokens: 2541748224 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.006972E+00 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 607/ 3100 | consumed samples: 1243136 | consumed tokens: 2545942528 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.895617E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 608/ 3100 | consumed samples: 1245184 | consumed tokens: 2550136832 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.002631E+00 | grad norm: 0.471 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 609/ 3100 | consumed samples: 1247232 | consumed tokens: 2554331136 | elapsed time per iteration (s): 139.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.954002E-01 | grad norm: 0.425 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.638 | TFLOPs: 149.43 | -[default7]: iteration 610/ 3100 | consumed samples: 1249280 | consumed tokens: 2558525440 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.007016E+00 | grad norm: 0.431 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 611/ 3100 | consumed samples: 1251328 | consumed tokens: 2562719744 | elapsed time per iteration (s): 141.14 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.007671E+00 | grad norm: 2.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.511 | TFLOPs: 148.13 | -[default7]: iteration 612/ 3100 | consumed samples: 1253376 | consumed tokens: 2566914048 | elapsed time per iteration (s): 140.29 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.010806E+00 | grad norm: 0.471 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.599 | TFLOPs: 149.03 | -[default7]: iteration 613/ 3100 | consumed samples: 1255424 | consumed tokens: 2571108352 | elapsed time per iteration (s): 139.98 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.002140E+00 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.631 | TFLOPs: 149.36 | -[default7]: iteration 614/ 3100 | consumed samples: 1257472 | consumed tokens: 2575302656 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.966283E-01 | grad norm: 0.425 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 615/ 3100 | consumed samples: 1259520 | consumed tokens: 2579496960 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.010104E+00 | grad norm: 0.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 616/ 3100 | consumed samples: 1261568 | consumed tokens: 2583691264 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.863082E-01 | grad norm: 0.487 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 617/ 3100 | consumed samples: 1263616 | consumed tokens: 2587885568 | elapsed time per iteration (s): 140.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.968076E-01 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.561 | TFLOPs: 148.64 | -[default7]: iteration 618/ 3100 | consumed samples: 1265664 | consumed tokens: 2592079872 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.000490E+00 | grad norm: 0.453 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 619/ 3100 | consumed samples: 1267712 | consumed tokens: 2596274176 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.970782E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 620/ 3100 | consumed samples: 1269760 | consumed tokens: 2600468480 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.002072E+00 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 621/ 3100 | consumed samples: 1271808 | consumed tokens: 2604662784 | elapsed time per iteration (s): 140.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.995710E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.590 | TFLOPs: 148.94 | -[default7]: iteration 622/ 3100 | consumed samples: 1273856 | consumed tokens: 2608857088 | elapsed time per iteration (s): 140.34 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.981472E-01 | grad norm: 0.431 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.593 | TFLOPs: 148.97 | -[default7]: iteration 623/ 3100 | consumed samples: 1275904 | consumed tokens: 2613051392 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.013494E+00 | grad norm: 0.893 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 624/ 3100 | consumed samples: 1277952 | consumed tokens: 2617245696 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.850723E-01 | grad norm: 0.439 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 625/ 3100 | consumed samples: 1280000 | consumed tokens: 2621440000 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.002878E+00 | grad norm: 0.499 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 626/ 3100 | consumed samples: 1282048 | consumed tokens: 2625634304 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.014546E+00 | grad norm: 0.480 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 627/ 3100 | consumed samples: 1284096 | consumed tokens: 2629828608 | elapsed time per iteration (s): 140.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.008642E+00 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.534 | TFLOPs: 148.37 | -[default7]: iteration 628/ 3100 | consumed samples: 1286144 | consumed tokens: 2634022912 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.880075E-01 | grad norm: 0.816 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 629/ 3100 | consumed samples: 1288192 | consumed tokens: 2638217216 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.924359E-01 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 630/ 3100 | consumed samples: 1290240 | consumed tokens: 2642411520 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.960424E-01 | grad norm: 0.575 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 631/ 3100 | consumed samples: 1292288 | consumed tokens: 2646605824 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.973553E-01 | grad norm: 0.550 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 632/ 3100 | consumed samples: 1294336 | consumed tokens: 2650800128 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.940726E-01 | grad norm: 0.738 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 633/ 3100 | consumed samples: 1296384 | consumed tokens: 2654994432 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.803886E-01 | grad norm: 0.448 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 634/ 3100 | consumed samples: 1298432 | consumed tokens: 2659188736 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.014657E+00 | grad norm: 0.468 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 635/ 3100 | consumed samples: 1300480 | consumed tokens: 2663383040 | elapsed time per iteration (s): 140.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.002855E+00 | grad norm: 0.482 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.575 | TFLOPs: 148.79 | -[default7]: iteration 636/ 3100 | consumed samples: 1302528 | consumed tokens: 2667577344 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.996338E-01 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 637/ 3100 | consumed samples: 1304576 | consumed tokens: 2671771648 | elapsed time per iteration (s): 140.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.940645E-01 | grad norm: 0.438 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.554 | TFLOPs: 148.57 | -[default7]: iteration 638/ 3100 | consumed samples: 1306624 | consumed tokens: 2675965952 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.001481E+00 | grad norm: 0.480 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 639/ 3100 | consumed samples: 1308672 | consumed tokens: 2680160256 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.008491E+00 | grad norm: 0.443 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 640/ 3100 | consumed samples: 1310720 | consumed tokens: 2684354560 | elapsed time per iteration (s): 139.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.981856E-01 | grad norm: 0.448 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.665 | TFLOPs: 149.71 | -[default7]: iteration 641/ 3100 | consumed samples: 1312768 | consumed tokens: 2688548864 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.877676E-01 | grad norm: 0.445 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 642/ 3100 | consumed samples: 1314816 | consumed tokens: 2692743168 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.873846E-01 | grad norm: 0.439 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 643/ 3100 | consumed samples: 1316864 | consumed tokens: 2696937472 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.999849E-01 | grad norm: 0.472 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.582 | TFLOPs: 148.86 | -[default7]: iteration 644/ 3100 | consumed samples: 1318912 | consumed tokens: 2701131776 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.790533E-01 | grad norm: 0.499 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 645/ 3100 | consumed samples: 1320960 | consumed tokens: 2705326080 | elapsed time per iteration (s): 141.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 1.012976E+00 | grad norm: 0.492 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.521 | TFLOPs: 148.24 | -[default7]: iteration 646/ 3100 | consumed samples: 1323008 | consumed tokens: 2709520384 | elapsed time per iteration (s): 141.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.860898E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.488 | TFLOPs: 147.90 | -[default7]: iteration 647/ 3100 | consumed samples: 1325056 | consumed tokens: 2713714688 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.992117E-01 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 648/ 3100 | consumed samples: 1327104 | consumed tokens: 2717908992 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.933904E-01 | grad norm: 0.459 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 649/ 3100 | consumed samples: 1329152 | consumed tokens: 2722103296 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.870589E-01 | grad norm: 0.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 650/ 3100 | consumed samples: 1331200 | consumed tokens: 2726297600 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.902743E-01 | grad norm: 0.457 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 651/ 3100 | consumed samples: 1333248 | consumed tokens: 2730491904 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.866619E-01 | grad norm: 0.568 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 652/ 3100 | consumed samples: 1335296 | consumed tokens: 2734686208 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.893770E-01 | grad norm: 0.475 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 653/ 3100 | consumed samples: 1337344 | consumed tokens: 2738880512 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.802227E-01 | grad norm: 0.477 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 654/ 3100 | consumed samples: 1339392 | consumed tokens: 2743074816 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.878392E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 655/ 3100 | consumed samples: 1341440 | consumed tokens: 2747269120 | elapsed time per iteration (s): 140.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.940200E-01 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.568 | TFLOPs: 148.71 | -[default7]: iteration 656/ 3100 | consumed samples: 1343488 | consumed tokens: 2751463424 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.800345E-01 | grad norm: 0.459 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 657/ 3100 | consumed samples: 1345536 | consumed tokens: 2755657728 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.743832E-01 | grad norm: 0.438 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 658/ 3100 | consumed samples: 1347584 | consumed tokens: 2759852032 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.913894E-01 | grad norm: 0.425 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 659/ 3100 | consumed samples: 1349632 | consumed tokens: 2764046336 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.855616E-01 | grad norm: 0.450 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 660/ 3100 | consumed samples: 1351680 | consumed tokens: 2768240640 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.851209E-01 | grad norm: 0.437 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 661/ 3100 | consumed samples: 1353728 | consumed tokens: 2772434944 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.781877E-01 | grad norm: 0.491 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 662/ 3100 | consumed samples: 1355776 | consumed tokens: 2776629248 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.733080E-01 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 663/ 3100 | consumed samples: 1357824 | consumed tokens: 2780823552 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.782169E-01 | grad norm: 1.021 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 664/ 3100 | consumed samples: 1359872 | consumed tokens: 2785017856 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.889561E-01 | grad norm: 0.489 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 665/ 3100 | consumed samples: 1361920 | consumed tokens: 2789212160 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.850926E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 666/ 3100 | consumed samples: 1363968 | consumed tokens: 2793406464 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.930356E-01 | grad norm: 0.467 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 667/ 3100 | consumed samples: 1366016 | consumed tokens: 2797600768 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.803570E-01 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 668/ 3100 | consumed samples: 1368064 | consumed tokens: 2801795072 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.805388E-01 | grad norm: 0.452 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 669/ 3100 | consumed samples: 1370112 | consumed tokens: 2805989376 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.655027E-01 | grad norm: 0.474 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 670/ 3100 | consumed samples: 1372160 | consumed tokens: 2810183680 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.781249E-01 | grad norm: 0.506 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 671/ 3100 | consumed samples: 1374208 | consumed tokens: 2814377984 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.704345E-01 | grad norm: 0.730 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 672/ 3100 | consumed samples: 1376256 | consumed tokens: 2818572288 | elapsed time per iteration (s): 141.29 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.805143E-01 | grad norm: 0.466 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.495 | TFLOPs: 147.97 | -[default7]: iteration 673/ 3100 | consumed samples: 1378304 | consumed tokens: 2822766592 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.920224E-01 | grad norm: 0.423 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 674/ 3100 | consumed samples: 1380352 | consumed tokens: 2826960896 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.838195E-01 | grad norm: 0.706 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.50 | -[default7]: iteration 675/ 3100 | consumed samples: 1382400 | consumed tokens: 2831155200 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.705421E-01 | grad norm: 0.493 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 676/ 3100 | consumed samples: 1384448 | consumed tokens: 2835349504 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.862552E-01 | grad norm: 0.477 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 677/ 3100 | consumed samples: 1386496 | consumed tokens: 2839543808 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.849824E-01 | grad norm: 0.470 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 678/ 3100 | consumed samples: 1388544 | consumed tokens: 2843738112 | elapsed time per iteration (s): 141.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.801395E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.491 | TFLOPs: 147.93 | -[default7]: iteration 679/ 3100 | consumed samples: 1390592 | consumed tokens: 2847932416 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.913411E-01 | grad norm: 0.471 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 680/ 3100 | consumed samples: 1392640 | consumed tokens: 2852126720 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.780481E-01 | grad norm: 0.471 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 681/ 3100 | consumed samples: 1394688 | consumed tokens: 2856321024 | elapsed time per iteration (s): 140.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.757339E-01 | grad norm: 0.858 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.559 | TFLOPs: 148.63 | -[default7]: iteration 682/ 3100 | consumed samples: 1396736 | consumed tokens: 2860515328 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.783306E-01 | grad norm: 0.507 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.49 | -[default7]: iteration 683/ 3100 | consumed samples: 1398784 | consumed tokens: 2864709632 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.723117E-01 | grad norm: 0.464 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 684/ 3100 | consumed samples: 1400832 | consumed tokens: 2868903936 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.830213E-01 | grad norm: 0.489 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 685/ 3100 | consumed samples: 1402880 | consumed tokens: 2873098240 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.902226E-01 | grad norm: 0.504 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 686/ 3100 | consumed samples: 1404928 | consumed tokens: 2877292544 | elapsed time per iteration (s): 141.31 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.700904E-01 | grad norm: 0.484 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.493 | TFLOPs: 147.95 | -[default7]: iteration 687/ 3100 | consumed samples: 1406976 | consumed tokens: 2881486848 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.875706E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 688/ 3100 | consumed samples: 1409024 | consumed tokens: 2885681152 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.891896E-01 | grad norm: 0.491 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 689/ 3100 | consumed samples: 1411072 | consumed tokens: 2889875456 | elapsed time per iteration (s): 140.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.837467E-01 | grad norm: 0.611 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.583 | TFLOPs: 148.87 | -[default7]: iteration 690/ 3100 | consumed samples: 1413120 | consumed tokens: 2894069760 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.814984E-01 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 691/ 3100 | consumed samples: 1415168 | consumed tokens: 2898264064 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.767624E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 692/ 3100 | consumed samples: 1417216 | consumed tokens: 2902458368 | elapsed time per iteration (s): 140.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.830289E-01 | grad norm: 0.491 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.581 | TFLOPs: 148.85 | -[default7]: iteration 693/ 3100 | consumed samples: 1419264 | consumed tokens: 2906652672 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.784164E-01 | grad norm: 0.447 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 694/ 3100 | consumed samples: 1421312 | consumed tokens: 2910846976 | elapsed time per iteration (s): 140.16 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.695111E-01 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.612 | TFLOPs: 149.17 | -[default7]: iteration 695/ 3100 | consumed samples: 1423360 | consumed tokens: 2915041280 | elapsed time per iteration (s): 140.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.734830E-01 | grad norm: 0.496 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.591 | TFLOPs: 148.95 | -[default7]: iteration 696/ 3100 | consumed samples: 1425408 | consumed tokens: 2919235584 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.712253E-01 | grad norm: 0.478 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 697/ 3100 | consumed samples: 1427456 | consumed tokens: 2923429888 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.585085E-01 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 698/ 3100 | consumed samples: 1429504 | consumed tokens: 2927624192 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.739147E-01 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 699/ 3100 | consumed samples: 1431552 | consumed tokens: 2931818496 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.715589E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 700/ 3100 | consumed samples: 1433600 | consumed tokens: 2936012800 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.597683E-01 | grad norm: 0.577 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 701/ 3100 | consumed samples: 1435648 | consumed tokens: 2940207104 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.722838E-01 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 702/ 3100 | consumed samples: 1437696 | consumed tokens: 2944401408 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.543227E-01 | grad norm: 2.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 703/ 3100 | consumed samples: 1439744 | consumed tokens: 2948595712 | elapsed time per iteration (s): 140.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.735118E-01 | grad norm: 0.500 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.559 | TFLOPs: 148.62 | -[default7]: iteration 704/ 3100 | consumed samples: 1441792 | consumed tokens: 2952790016 | elapsed time per iteration (s): 141.35 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.728131E-01 | grad norm: 4.245 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.489 | TFLOPs: 147.91 | -[default7]: iteration 705/ 3100 | consumed samples: 1443840 | consumed tokens: 2956984320 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.677306E-01 | grad norm: 0.487 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 706/ 3100 | consumed samples: 1445888 | consumed tokens: 2961178624 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.850529E-01 | grad norm: 0.547 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 707/ 3100 | consumed samples: 1447936 | consumed tokens: 2965372928 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.751300E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.582 | TFLOPs: 148.86 | -[default7]: iteration 708/ 3100 | consumed samples: 1449984 | consumed tokens: 2969567232 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.636545E-01 | grad norm: 0.596 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 709/ 3100 | consumed samples: 1452032 | consumed tokens: 2973761536 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.847851E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 710/ 3100 | consumed samples: 1454080 | consumed tokens: 2977955840 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.638894E-01 | grad norm: 0.460 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 711/ 3100 | consumed samples: 1456128 | consumed tokens: 2982150144 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.694620E-01 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 712/ 3100 | consumed samples: 1458176 | consumed tokens: 2986344448 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.607476E-01 | grad norm: 0.439 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 713/ 3100 | consumed samples: 1460224 | consumed tokens: 2990538752 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.751613E-01 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 714/ 3100 | consumed samples: 1462272 | consumed tokens: 2994733056 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.710302E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 715/ 3100 | consumed samples: 1464320 | consumed tokens: 2998927360 | elapsed time per iteration (s): 141.06 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.721834E-01 | grad norm: 0.455 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.518 | TFLOPs: 148.21 | -[default7]: iteration 716/ 3100 | consumed samples: 1466368 | consumed tokens: 3003121664 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.720158E-01 | grad norm: 0.450 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 717/ 3100 | consumed samples: 1468416 | consumed tokens: 3007315968 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.637253E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 718/ 3100 | consumed samples: 1470464 | consumed tokens: 3011510272 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.607047E-01 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 719/ 3100 | consumed samples: 1472512 | consumed tokens: 3015704576 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.712372E-01 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 720/ 3100 | consumed samples: 1474560 | consumed tokens: 3019898880 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.668542E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 721/ 3100 | consumed samples: 1476608 | consumed tokens: 3024093184 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.570224E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.499 | TFLOPs: 148.02 | -[default7]: iteration 722/ 3100 | consumed samples: 1478656 | consumed tokens: 3028287488 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.508538E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 723/ 3100 | consumed samples: 1480704 | consumed tokens: 3032481792 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.647309E-01 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 724/ 3100 | consumed samples: 1482752 | consumed tokens: 3036676096 | elapsed time per iteration (s): 141.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.757177E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.484 | TFLOPs: 147.86 | -[default7]: iteration 725/ 3100 | consumed samples: 1484800 | consumed tokens: 3040870400 | elapsed time per iteration (s): 141.28 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.654821E-01 | grad norm: 1.657 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.496 | TFLOPs: 147.98 | -[default7]: iteration 726/ 3100 | consumed samples: 1486848 | consumed tokens: 3045064704 | elapsed time per iteration (s): 141.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.549273E-01 | grad norm: 0.475 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.486 | TFLOPs: 147.88 | -[default7]: iteration 727/ 3100 | consumed samples: 1488896 | consumed tokens: 3049259008 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.553468E-01 | grad norm: 0.451 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 728/ 3100 | consumed samples: 1490944 | consumed tokens: 3053453312 | elapsed time per iteration (s): 141.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.663463E-01 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 729/ 3100 | consumed samples: 1492992 | consumed tokens: 3057647616 | elapsed time per iteration (s): 140.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.588075E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.580 | TFLOPs: 148.84 | -[default7]: iteration 730/ 3100 | consumed samples: 1495040 | consumed tokens: 3061841920 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.583896E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 731/ 3100 | consumed samples: 1497088 | consumed tokens: 3066036224 | elapsed time per iteration (s): 140.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.594564E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.588 | TFLOPs: 148.92 | -[default7]: iteration 732/ 3100 | consumed samples: 1499136 | consumed tokens: 3070230528 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.715738E-01 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.62 | -[default7]: iteration 733/ 3100 | consumed samples: 1501184 | consumed tokens: 3074424832 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.715696E-01 | grad norm: 0.465 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 734/ 3100 | consumed samples: 1503232 | consumed tokens: 3078619136 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.529501E-01 | grad norm: 0.459 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 735/ 3100 | consumed samples: 1505280 | consumed tokens: 3082813440 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.602795E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 736/ 3100 | consumed samples: 1507328 | consumed tokens: 3087007744 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.549717E-01 | grad norm: 0.449 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 737/ 3100 | consumed samples: 1509376 | consumed tokens: 3091202048 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.583347E-01 | grad norm: 0.547 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 738/ 3100 | consumed samples: 1511424 | consumed tokens: 3095396352 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.507765E-01 | grad norm: 0.479 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 739/ 3100 | consumed samples: 1513472 | consumed tokens: 3099590656 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.498522E-01 | grad norm: 0.431 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 740/ 3100 | consumed samples: 1515520 | consumed tokens: 3103784960 | elapsed time per iteration (s): 140.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.541953E-01 | grad norm: 0.452 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.565 | TFLOPs: 148.69 | -[default7]: iteration 741/ 3100 | consumed samples: 1517568 | consumed tokens: 3107979264 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.519040E-01 | grad norm: 0.613 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 742/ 3100 | consumed samples: 1519616 | consumed tokens: 3112173568 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.525378E-01 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 743/ 3100 | consumed samples: 1521664 | consumed tokens: 3116367872 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.363645E-01 | grad norm: 0.542 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.84 | -[default7]: iteration 744/ 3100 | consumed samples: 1523712 | consumed tokens: 3120562176 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.727218E-01 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 745/ 3100 | consumed samples: 1525760 | consumed tokens: 3124756480 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.500061E-01 | grad norm: 0.458 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 746/ 3100 | consumed samples: 1527808 | consumed tokens: 3128950784 | elapsed time per iteration (s): 140.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.593070E-01 | grad norm: 0.502 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.566 | TFLOPs: 148.69 | -[default0]:saving checkpoint at iteration 747 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-09 03:31:48,662] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step747 is begin to save! -[default4]:[2022-09-09 03:31:48,740] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_55-model_00-model_states.pt... -[default7]: iteration 747/ 3100 | consumed samples: 1529856 | consumed tokens: 3133145088 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.592397E-01 | grad norm: 0.423 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default4]:[2022-09-09 03:31:48,710] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_15-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,740] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_54-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,710] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_14-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,806] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_17-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_05-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,824] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_58-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,825] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_53-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,824] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_71_model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_60-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,831] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_09-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,830] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_72-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,831] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_62-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,841] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_47-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,841] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_19-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_45-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_34-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_71-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,845] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_68-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_35-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_13-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,826] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_28-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,831] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_08-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_51-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_20-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_29-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,842] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_42-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_21-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_03-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_44-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_61-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_41-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_26-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,845] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_69-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_57-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_40-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,830] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_24-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_12-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_70-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,825] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_59-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,806] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_16-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_50-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,892] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_67-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,841] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_10-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_27-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,830] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_46-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_38-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_39-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,825] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_52-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,893] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_32-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,830] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_25-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_11-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,841] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_63-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,841] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_18-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,841] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_49-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_56-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,883] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_30-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_48-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,826] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_04-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,838] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_71_model_states.pt. -[default0]:[2022-09-09 03:31:48,844] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_01-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,883] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_64-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,883] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_65-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,892] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_33-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,913] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_07-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,883] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_31-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_43-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,892] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_66-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,913] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_06-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,923] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_23-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,913] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_36-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:48,923] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_22-model_00-model_states.pt... -[default4]:[2022-09-09 03:31:48,913] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_37-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:52,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_72-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,072] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_74-model_00-model_states.pt... -[default0]:[2022-09-09 03:31:52,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_74-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,077] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_70_model_states.pt... -[default0]:[2022-09-09 03:31:52,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_70_model_states.pt. -[default4]:[2022-09-09 03:31:52,125] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_19-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,126] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_17_model_states.pt... -[default4]:[2022-09-09 03:31:52,128] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_17_model_states.pt. -[default0]:[2022-09-09 03:31:52,312] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_52-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,312] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_50_model_states.pt... -[default4]:[2022-09-09 03:31:52,298] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_17-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,299] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_15_model_states.pt... -[default4]:[2022-09-09 03:31:52,302] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_15_model_states.pt. -[default4]:[2022-09-09 03:31:52,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_71-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,305] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_69_model_states.pt... -[default4]:[2022-09-09 03:31:52,309] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_69_model_states.pt. -[default4]:[2022-09-09 03:31:52,338] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_59-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,338] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_57_model_states.pt... -[default4]:[2022-09-09 03:31:52,341] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_57_model_states.pt. -[default0]:[2022-09-09 03:31:52,391] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_50-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,391] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_48_model_states.pt... -[default0]:[2022-09-09 03:31:52,394] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_48_model_states.pt. -[default0]:[2022-09-09 03:31:52,315] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_50_model_states.pt. -[default0]:[2022-09-09 03:31:52,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_18-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,408] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_16_model_states.pt... -[default0]:[2022-09-09 03:31:52,409] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_16_model_states.pt. -[default0]:[2022-09-09 03:31:52,426] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_58-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,426] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_56_model_states.pt... -[default0]:[2022-09-09 03:31:52,430] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_56_model_states.pt. -[default0]:[2022-09-09 03:31:52,444] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_42-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,444] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_40_model_states.pt... -[default0]:[2022-09-09 03:31:52,447] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_40_model_states.pt. -[default0]:[2022-09-09 03:31:52,413] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_26-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,414] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_24_model_states.pt... -[default0]:[2022-09-09 03:31:52,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_24_model_states.pt. -[default4]:[2022-09-09 03:31:52,485] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_45-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,486] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_43_model_states.pt... -[default4]:[2022-09-09 03:31:52,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_43_model_states.pt. -[default4]:[2022-09-09 03:31:52,526] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_51-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,526] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_49_model_states.pt... -[default4]:[2022-09-09 03:31:52,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_49_model_states.pt. -[default0]:[2022-09-09 03:31:52,514] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_44-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,514] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_42_model_states.pt... -[default0]:[2022-09-09 03:31:52,517] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_42_model_states.pt. -[default0]:[2022-09-09 03:31:52,513] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_70-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,514] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_68_model_states.pt... -[default0]:[2022-09-09 03:31:52,517] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_68_model_states.pt. -[default0]:[2022-09-09 03:31:52,557] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_16-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,557] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_14_model_states.pt... -[default0]:[2022-09-09 03:31:52,560] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_14_model_states.pt. -[default4]:[2022-09-09 03:31:52,585] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_27-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,586] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_25_model_states.pt... -[default4]:[2022-09-09 03:31:52,588] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_25_model_states.pt. -[default4]:[2022-09-09 03:31:52,535] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_49-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,535] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_47_model_states.pt... -[default4]:[2022-09-09 03:31:52,542] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_47_model_states.pt. -[default0]:[2022-09-09 03:31:52,623] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_30-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,623] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_28_model_states.pt... -[default0]:[2022-09-09 03:31:52,539] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_48-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,540] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_46_model_states.pt... -[default0]:[2022-09-09 03:31:52,544] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_46_model_states.pt. -[default4]:[2022-09-09 03:31:52,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_55-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,616] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_53_model_states.pt... -[default4]:[2022-09-09 03:31:52,618] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_53_model_states.pt. -[default4]:[2022-09-09 03:31:52,627] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_43-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,627] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_41_model_states.pt... -[default4]:[2022-09-09 03:31:52,630] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_41_model_states.pt. -[default0]:[2022-09-09 03:31:52,674] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_06-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,674] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_04_model_states.pt... -[default4]:[2022-09-09 03:31:52,664] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_03-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,665] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_01_model_states.pt... -[default4]:[2022-09-09 03:31:52,667] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_01_model_states.pt. -[default0]:[2022-09-09 03:31:52,658] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_54-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,659] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_52_model_states.pt... -[default0]:[2022-09-09 03:31:52,660] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_52_model_states.pt. -[default0]:[2022-09-09 03:31:52,627] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_28_model_states.pt. -[default4]:[2022-09-09 03:31:52,640] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_05-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,641] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_03_model_states.pt... -[default4]:[2022-09-09 03:31:52,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_03_model_states.pt. -[default4]:[2022-09-09 03:31:52,649] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_53-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,650] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_51_model_states.pt... -[default4]:[2022-09-09 03:31:52,651] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_51_model_states.pt. -[default0]:[2022-09-09 03:31:52,727] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_64-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,728] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_62_model_states.pt... -[default0]:[2022-09-09 03:31:52,731] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_62_model_states.pt. -[default4]:[2022-09-09 03:31:52,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_13-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,685] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_11_model_states.pt... -[default4]:[2022-09-09 03:31:52,688] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_11_model_states.pt. -[default0]:[2022-09-09 03:31:52,676] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_04_model_states.pt. -[default4]:[2022-09-09 03:31:52,772] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_15-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,773] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_13_model_states.pt... -[default4]:[2022-09-09 03:31:52,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_13_model_states.pt. -[default0]:[2022-09-09 03:31:52,764] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_12-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,765] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_10_model_states.pt... -[default0]:[2022-09-09 03:31:52,767] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_10_model_states.pt. -[default0]:[2022-09-09 03:31:52,744] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_14-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,744] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_12_model_states.pt... -[default0]:[2022-09-09 03:31:52,748] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_12_model_states.pt. -[default0]:[2022-09-09 03:31:52,803] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_04-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,803] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_02_model_states.pt... -[default0]:[2022-09-09 03:31:52,805] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_02_model_states.pt. -[default4]:[2022-09-09 03:31:52,803] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_65-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,803] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_63_model_states.pt... -[default4]:[2022-09-09 03:31:52,809] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_63_model_states.pt. -[default4]:[2022-09-09 03:31:52,841] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_33-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,841] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_31_model_states.pt... -[default4]:[2022-09-09 03:31:52,843] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_31_model_states.pt. -[default4]:[2022-09-09 03:31:52,761] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_07-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,761] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_05_model_states.pt... -[default4]:[2022-09-09 03:31:52,763] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_05_model_states.pt. -[default4]:[2022-09-09 03:31:52,763] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_31-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,763] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_29_model_states.pt... -[default4]:[2022-09-09 03:31:52,766] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_29_model_states.pt. -[default4]:[2022-09-09 03:31:52,843] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_57-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,843] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_55_model_states.pt... -[default4]:[2022-09-09 03:31:52,846] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_55_model_states.pt. -[default0]:[2022-09-09 03:31:52,819] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_38-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,819] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_36_model_states.pt... -[default0]:[2022-09-09 03:31:52,824] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_36_model_states.pt. -[default0]:[2022-09-09 03:31:52,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_32-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,838] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_30_model_states.pt... -[default0]:[2022-09-09 03:31:52,840] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_30_model_states.pt. -[default0]:[2022-09-09 03:31:52,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_56-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,855] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_54_model_states.pt... -[default0]:[2022-09-09 03:31:52,857] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_54_model_states.pt. -[default0]:[2022-09-09 03:31:52,892] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_34-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,892] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_32_model_states.pt... -[default0]:[2022-09-09 03:31:52,895] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_32_model_states.pt. -[default4]:[2022-09-09 03:31:52,907] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_35-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,907] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_33_model_states.pt... -[default4]:[2022-09-09 03:31:52,909] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_33_model_states.pt. -[default0]:[2022-09-09 03:31:52,930] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_28-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,931] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_26_model_states.pt... -[default0]:[2022-09-09 03:31:52,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_26_model_states.pt. -[default0]:[2022-09-09 03:31:52,931] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_20-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,931] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_18_model_states.pt... -[default0]:[2022-09-09 03:31:52,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_18_model_states.pt. -[default4]:[2022-09-09 03:31:52,921] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_29-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,921] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_27_model_states.pt... -[default4]:[2022-09-09 03:31:52,923] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_27_model_states.pt. -[default4]:[2022-09-09 03:31:52,941] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_21-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,941] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_19_model_states.pt... -[default4]:[2022-09-09 03:31:52,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_19_model_states.pt. -[default4]:[2022-09-09 03:31:52,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_61-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,973] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_59_model_states.pt... -[default4]:[2022-09-09 03:31:52,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_59_model_states.pt. -[default0]:[2022-09-09 03:31:52,984] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_36-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,985] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_34_model_states.pt... -[default0]:[2022-09-09 03:31:52,987] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_34_model_states.pt. -[default4]:[2022-09-09 03:31:52,942] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_69-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,942] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_67_model_states.pt... -[default4]:[2022-09-09 03:31:52,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_67_model_states.pt. -[default0]:[2022-09-09 03:31:52,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_24-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,934] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_22_model_states.pt... -[default0]:[2022-09-09 03:31:52,936] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_22_model_states.pt. -[default4]:[2022-09-09 03:31:52,977] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_39-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,978] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_37_model_states.pt... -[default4]:[2022-09-09 03:31:52,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_37_model_states.pt. -[default4]:[2022-09-09 03:31:52,988] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_25-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,988] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_23_model_states.pt... -[default4]:[2022-09-09 03:31:52,990] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_23_model_states.pt. -[default4]:[2022-09-09 03:31:52,942] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_11-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,942] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_09_model_states.pt... -[default4]:[2022-09-09 03:31:52,948] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_09_model_states.pt. -[default4]:[2022-09-09 03:31:52,948] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_63-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,948] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_61_model_states.pt... -[default4]:[2022-09-09 03:31:52,950] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_61_model_states.pt. -[default0]:[2022-09-09 03:31:52,970] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_60-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:52,971] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_58_model_states.pt... -[default0]:[2022-09-09 03:31:52,974] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_58_model_states.pt. -[default4]:[2022-09-09 03:31:53,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_47-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:53,036] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_45_model_states.pt... -[default4]:[2022-09-09 03:31:53,038] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_45_model_states.pt. -[default0]:[2022-09-09 03:31:53,038] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_68-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:53,038] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_66_model_states.pt... -[default0]:[2022-09-09 03:31:53,040] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_66_model_states.pt. -[default4]:[2022-09-09 03:31:53,075] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_41-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:53,075] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_39_model_states.pt... -[default4]:[2022-09-09 03:31:53,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_39_model_states.pt. -[default0]:[2022-09-09 03:31:53,053] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_40-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:53,053] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_38_model_states.pt... -[default0]:[2022-09-09 03:31:53,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_38_model_states.pt. -[default0]:[2022-09-09 03:31:53,007] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_22-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:53,008] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_20_model_states.pt... -[default0]:[2022-09-09 03:31:53,010] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_20_model_states.pt. -[default4]:[2022-09-09 03:31:52,998] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_37-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:52,998] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_35_model_states.pt... -[default4]:[2022-09-09 03:31:53,003] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_35_model_states.pt. -[default0]:[2022-09-09 03:31:53,006] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_10-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:53,006] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_08_model_states.pt... -[default0]:[2022-09-09 03:31:53,008] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_08_model_states.pt. -[default0]:[2022-09-09 03:31:53,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_46-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:53,020] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_44_model_states.pt... -[default0]:[2022-09-09 03:31:53,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_44_model_states.pt. -[default4]:[2022-09-09 03:31:53,135] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_09-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:53,136] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_07_model_states.pt... -[default4]:[2022-09-09 03:31:53,138] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_07_model_states.pt. -[default0]:[2022-09-09 03:31:53,124] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_62-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:53,125] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_60_model_states.pt... -[default0]:[2022-09-09 03:31:53,126] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_60_model_states.pt. -[default0]:[2022-09-09 03:31:53,123] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_08-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:53,124] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_06_model_states.pt... -[default0]:[2022-09-09 03:31:53,127] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_06_model_states.pt. -[default0]:[2022-09-09 03:31:53,096] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_66-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:53,097] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_64_model_states.pt... -[default0]:[2022-09-09 03:31:53,099] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_64_model_states.pt. -[default4]:[2022-09-09 03:31:53,086] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_23-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:53,087] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_21_model_states.pt... -[default4]:[2022-09-09 03:31:53,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_21_model_states.pt. -[default4]:[2022-09-09 03:31:53,144] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_67-model_00-model_states.pt. -[default4]:[2022-09-09 03:31:53,144] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_65_model_states.pt... -[default4]:[2022-09-09 03:31:53,146] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_65_model_states.pt. -[default0]:[2022-09-09 03:31:53,657] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/layer_01-model_00-model_states.pt. -[default0]:[2022-09-09 03:31:53,658] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_00_model_states.pt -[default0]:[2022-09-09 03:31:53,658] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_00_model_states.pt... -[default0]:[2022-09-09 03:31:53,659] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/mp_rank_00_model_states.pt. -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default1]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default0]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default6]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default5]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default2]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default3]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default4]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default7]:[2022-09-09 03:31:53,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default5]:[2022-09-09 03:32:01,780] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-09 03:32:01,781] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default0]:[2022-09-09 03:32:03,162] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-09 03:32:03,163] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default4]:[2022-09-09 03:32:03,182] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-09 03:32:03,182] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default1]:[2022-09-09 03:32:03,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-09 03:32:03,204] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default3]:[2022-09-09 03:32:03,477] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-09 03:32:03,477] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default7]:[2022-09-09 03:32:03,526] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-09 03:32:03,527] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default7]:[2022-09-09 03:32:03,440] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-09 03:32:03,440] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default4]:[2022-09-09 03:32:03,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-09 03:32:03,488] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default3]:[2022-09-09 03:32:03,529] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-09 03:32:03,529] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default2]:[2022-09-09 03:32:03,649] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-09 03:32:03,649] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default4]:[2022-09-09 03:32:03,723] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-09 03:32:03,723] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default0]:[2022-09-09 03:32:03,818] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-09 03:32:03,818] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default1]:[2022-09-09 03:32:03,804] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-09 03:32:03,804] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default4]:[2022-09-09 03:32:03,785] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-09 03:32:03,785] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default4]:[2022-09-09 03:32:03,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-09 03:32:03,831] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default0]:[2022-09-09 03:32:03,829] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-09 03:32:03,829] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default3]:[2022-09-09 03:32:03,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-09 03:32:03,870] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default7]:[2022-09-09 03:32:03,879] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-09 03:32:03,880] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default7]:[2022-09-09 03:32:04,040] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-09 03:32:04,040] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default3]:[2022-09-09 03:32:04,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-09 03:32:04,097] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default2]:[2022-09-09 03:32:04,071] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-09 03:32:04,071] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default6]:[2022-09-09 03:32:04,154] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-09 03:32:04,154] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default5]:[2022-09-09 03:32:04,164] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-09 03:32:04,164] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default0]:[2022-09-09 03:32:04,260] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-09 03:32:04,260] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default3]:[2022-09-09 03:32:04,286] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-09 03:32:04,286] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default5]:[2022-09-09 03:32:04,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-09 03:32:04,335] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default1]:[2022-09-09 03:32:04,342] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-09 03:32:04,342] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default3]:[2022-09-09 03:32:04,432] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-09 03:32:04,432] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default0]:[2022-09-09 03:32:04,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-09 03:32:04,416] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default0]:[2022-09-09 03:32:04,527] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-09 03:32:04,528] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default2]:[2022-09-09 03:32:04,596] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-09 03:32:04,597] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default2]:[2022-09-09 03:32:04,619] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-09 03:32:04,619] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default1]:[2022-09-09 03:32:04,604] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-09 03:32:04,604] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default2]:[2022-09-09 03:32:04,723] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-09 03:32:04,724] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default6]:[2022-09-09 03:32:04,668] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-09 03:32:04,669] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default0]:[2022-09-09 03:32:04,744] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-09 03:32:04,744] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default5]:[2022-09-09 03:32:04,752] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-09 03:32:04,752] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default1]:[2022-09-09 03:32:04,774] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-09 03:32:04,774] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default5]:[2022-09-09 03:32:04,835] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-09 03:32:04,835] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default6]:[2022-09-09 03:32:04,832] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-09 03:32:04,832] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default7]:[2022-09-09 03:32:04,851] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-09 03:32:04,851] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default6]:[2022-09-09 03:32:04,798] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-09 03:32:04,798] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default3]:[2022-09-09 03:32:04,891] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-09 03:32:04,891] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default3]:[2022-09-09 03:32:04,897] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-09 03:32:04,897] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default6]:[2022-09-09 03:32:04,882] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-09 03:32:04,882] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default6]:[2022-09-09 03:32:04,875] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-09 03:32:04,875] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default1]:[2022-09-09 03:32:04,916] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-09 03:32:04,916] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default3]:[2022-09-09 03:32:04,893] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-09 03:32:04,893] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default4]:[2022-09-09 03:32:04,906] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-09 03:32:04,907] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default0]:[2022-09-09 03:32:04,987] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-09 03:32:04,987] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default2]:[2022-09-09 03:32:05,009] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-09 03:32:05,009] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default4]:[2022-09-09 03:32:04,959] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-09 03:32:04,959] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default2]:[2022-09-09 03:32:05,013] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-09 03:32:05,013] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default0]:[2022-09-09 03:32:05,060] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-09 03:32:05,060] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default1]:[2022-09-09 03:32:05,091] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-09 03:32:05,091] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default5]:[2022-09-09 03:32:05,132] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-09 03:32:05,132] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default7]:[2022-09-09 03:32:05,176] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-09 03:32:05,176] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default3]:[2022-09-09 03:32:05,166] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-09 03:32:05,166] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default3]:[2022-09-09 03:32:05,139] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-09 03:32:05,139] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default0]:[2022-09-09 03:32:05,172] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-09 03:32:05,172] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default3]:[2022-09-09 03:32:05,192] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-09 03:32:05,192] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default7]:[2022-09-09 03:32:05,131] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-09 03:32:05,131] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default1]:[2022-09-09 03:32:05,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-09 03:32:05,221] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default6]:[2022-09-09 03:32:05,231] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-09 03:32:05,231] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default3]:[2022-09-09 03:32:05,176] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-09 03:32:05,176] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default5]:[2022-09-09 03:32:05,258] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-09 03:32:05,259] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default2]:[2022-09-09 03:32:05,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-09 03:32:05,222] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default7]:[2022-09-09 03:32:05,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-09 03:32:05,262] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default1]:[2022-09-09 03:32:05,266] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-09 03:32:05,266] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default3]:[2022-09-09 03:32:05,294] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-09 03:32:05,294] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default2]:[2022-09-09 03:32:05,240] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-09 03:32:05,240] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default6]:[2022-09-09 03:32:05,281] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-09 03:32:05,281] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default5]:[2022-09-09 03:32:05,306] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-09 03:32:05,306] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default7]:[2022-09-09 03:32:05,345] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-09 03:32:05,346] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default1]:[2022-09-09 03:32:05,325] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-09 03:32:05,325] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default3]:[2022-09-09 03:32:05,398] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-09 03:32:05,398] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default3]:[2022-09-09 03:32:05,408] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-09 03:32:05,408] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default7]:[2022-09-09 03:32:05,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-09 03:32:05,455] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default4]:[2022-09-09 03:32:05,389] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-09 03:32:05,389] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default7]:[2022-09-09 03:32:05,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-09 03:32:05,386] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default2]:[2022-09-09 03:32:05,452] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-09 03:32:05,452] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default5]:[2022-09-09 03:32:05,471] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-09 03:32:05,471] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default2]:[2022-09-09 03:32:05,424] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-09 03:32:05,424] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default1]:[2022-09-09 03:32:05,500] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-09 03:32:05,501] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default5]:[2022-09-09 03:32:05,444] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-09 03:32:05,444] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default3]:[2022-09-09 03:32:05,468] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-09 03:32:05,468] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default4]:[2022-09-09 03:32:05,504] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-09 03:32:05,504] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default4]:[2022-09-09 03:32:05,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-09 03:32:05,488] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default2]:[2022-09-09 03:32:05,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-09 03:32:05,547] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default0]:[2022-09-09 03:32:05,531] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-09 03:32:05,531] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default7]:[2022-09-09 03:32:05,564] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-09 03:32:05,564] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default4]:[2022-09-09 03:32:05,498] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-09 03:32:05,498] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default2]:[2022-09-09 03:32:05,506] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-09 03:32:05,506] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default2]:[2022-09-09 03:32:05,504] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-09 03:32:05,505] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default2]:[2022-09-09 03:32:05,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-09 03:32:05,559] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default5]:[2022-09-09 03:32:05,583] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-09 03:32:05,583] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default3]:[2022-09-09 03:32:05,660] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-09 03:32:05,660] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default7]:[2022-09-09 03:32:05,646] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-09 03:32:05,646] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default1]:[2022-09-09 03:32:05,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-09 03:32:05,647] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default5]:[2022-09-09 03:32:05,689] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-09 03:32:05,689] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default1]:[2022-09-09 03:32:05,661] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-09 03:32:05,661] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default7]:[2022-09-09 03:32:05,743] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-09 03:32:05,743] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default5]:[2022-09-09 03:32:05,732] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-09 03:32:05,732] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default1]:[2022-09-09 03:32:05,761] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-09 03:32:05,761] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default7]:[2022-09-09 03:32:05,717] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-09 03:32:05,717] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default0]:[2022-09-09 03:32:05,682] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-09 03:32:05,682] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default6]:[2022-09-09 03:32:05,762] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-09 03:32:05,762] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default1]:[2022-09-09 03:32:05,758] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-09 03:32:05,758] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default2]:[2022-09-09 03:32:05,792] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-09 03:32:05,793] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default0]:[2022-09-09 03:32:05,765] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-09 03:32:05,765] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default6]:[2022-09-09 03:32:05,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-09 03:32:05,808] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default4]:[2022-09-09 03:32:05,767] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-09 03:32:05,767] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default6]:[2022-09-09 03:32:05,807] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-09 03:32:05,808] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default3]:[2022-09-09 03:32:05,825] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-09 03:32:05,825] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default6]:[2022-09-09 03:32:05,826] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-09 03:32:05,826] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default5]:[2022-09-09 03:32:05,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-09 03:32:05,865] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default2]:[2022-09-09 03:32:05,849] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-09 03:32:05,849] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default5]:[2022-09-09 03:32:05,807] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-09 03:32:05,807] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default7]:[2022-09-09 03:32:05,843] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-09 03:32:05,843] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default4]:[2022-09-09 03:32:05,855] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-09 03:32:05,855] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default4]:[2022-09-09 03:32:05,880] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-09 03:32:05,881] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default3]:[2022-09-09 03:32:05,838] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-09 03:32:05,838] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default2]:[2022-09-09 03:32:05,875] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-09 03:32:05,875] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default5]:[2022-09-09 03:32:05,946] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-09 03:32:05,946] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default6]:[2022-09-09 03:32:05,855] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-09 03:32:05,855] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default0]:[2022-09-09 03:32:05,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-09 03:32:05,865] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default0]:[2022-09-09 03:32:05,907] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-09 03:32:05,907] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default2]:[2022-09-09 03:32:05,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-09 03:32:05,947] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default5]:[2022-09-09 03:32:05,902] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-09 03:32:05,902] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default1]:[2022-09-09 03:32:05,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-09 03:32:05,933] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default5]:[2022-09-09 03:32:05,957] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-09 03:32:05,957] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default0]:[2022-09-09 03:32:05,979] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-09 03:32:05,979] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default4]:[2022-09-09 03:32:06,001] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-09 03:32:06,001] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default6]:[2022-09-09 03:32:05,955] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-09 03:32:05,955] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default1]:[2022-09-09 03:32:05,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-09 03:32:05,973] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default1]:[2022-09-09 03:32:06,034] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-09 03:32:06,034] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default0]:[2022-09-09 03:32:06,029] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-09 03:32:06,029] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default7]:[2022-09-09 03:32:06,029] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-09 03:32:06,029] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default1]:[2022-09-09 03:32:06,031] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-09 03:32:06,031] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default7]:[2022-09-09 03:32:06,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-09 03:32:06,023] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default6]:[2022-09-09 03:32:06,078] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-09 03:32:06,078] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default4]:[2022-09-09 03:32:06,034] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-09 03:32:06,034] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default6]:[2022-09-09 03:32:06,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-09 03:32:06,023] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default4]:[2022-09-09 03:32:06,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-09 03:32:06,072] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default5]:[2022-09-09 03:32:06,158] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-09 03:32:06,158] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default4]:[2022-09-09 03:32:06,119] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-09 03:32:06,119] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default4]:[2022-09-09 03:32:06,102] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-09 03:32:06,103] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default2]:[2022-09-09 03:32:06,117] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-09 03:32:06,117] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default0]:[2022-09-09 03:32:06,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-09 03:32:06,097] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default6]:[2022-09-09 03:32:06,129] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-09 03:32:06,129] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default4]:[2022-09-09 03:32:06,129] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-09 03:32:06,129] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default7]:[2022-09-09 03:32:06,225] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-09 03:32:06,225] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default0]:[2022-09-09 03:32:06,143] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-09 03:32:06,143] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default2]:[2022-09-09 03:32:06,152] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-09 03:32:06,152] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default7]:[2022-09-09 03:32:06,195] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-09 03:32:06,195] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default0]:[2022-09-09 03:32:06,230] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-09 03:32:06,230] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default7]:[2022-09-09 03:32:06,206] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-09 03:32:06,206] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default6]:[2022-09-09 03:32:06,264] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-09 03:32:06,265] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default5]:[2022-09-09 03:32:06,261] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-09 03:32:06,261] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default4]:[2022-09-09 03:32:06,283] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-09 03:32:06,283] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default6]:[2022-09-09 03:32:06,300] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-09 03:32:06,300] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default1]:[2022-09-09 03:32:06,300] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-09 03:32:06,301] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default6]:[2022-09-09 03:32:06,353] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-09 03:32:06,354] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default6]:[2022-09-09 03:32:06,377] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-09 03:32:06,377] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default3]:[2022-09-09 03:32:06,449] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-09 03:32:06,449] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default6]:[2022-09-09 03:32:06,428] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-09 03:32:06,429] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default7]:[2022-09-09 03:32:06,423] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-09 03:32:06,423] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default1]:[2022-09-09 03:32:06,484] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-09 03:32:06,484] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default0]:[2022-09-09 03:32:06,467] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-09 03:32:06,467] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default3]:[2022-09-09 03:32:06,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-09 03:32:06,528] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default5]:[2022-09-09 03:32:06,572] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-09 03:32:06,572] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default4]:[2022-09-09 03:32:06,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-09 03:32:06,576] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default5]:[2022-09-09 03:32:06,526] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-09 03:32:06,526] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default6]:[2022-09-09 03:32:06,565] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-09 03:32:06,565] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default0]:[2022-09-09 03:32:06,702] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-09 03:32:06,702] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default3]:[2022-09-09 03:32:06,720] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-09 03:32:06,720] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default0]:[2022-09-09 03:32:06,765] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-09 03:32:06,765] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default6]:[2022-09-09 03:32:06,794] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-09 03:32:06,794] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default4]:[2022-09-09 03:32:06,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-09 03:32:06,800] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default4]:[2022-09-09 03:32:06,859] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-09 03:32:06,859] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default3]:[2022-09-09 03:32:06,804] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-09 03:32:06,804] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default0]:[2022-09-09 03:32:06,867] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-09 03:32:06,868] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default2]:[2022-09-09 03:32:06,899] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-09 03:32:06,899] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default3]:[2022-09-09 03:32:06,918] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-09 03:32:06,918] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default0]:[2022-09-09 03:32:06,869] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-09 03:32:06,869] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default6]:[2022-09-09 03:32:06,908] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-09 03:32:06,908] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default2]:[2022-09-09 03:32:06,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-09 03:32:06,993] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default7]:[2022-09-09 03:32:07,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-09 03:32:07,004] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default1]:[2022-09-09 03:32:06,952] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-09 03:32:06,952] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default1]:[2022-09-09 03:32:06,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-09 03:32:06,973] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default7]:[2022-09-09 03:32:06,959] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-09 03:32:06,959] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default4]:[2022-09-09 03:32:07,044] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-09 03:32:07,044] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default1]:[2022-09-09 03:32:06,992] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-09 03:32:06,992] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default1]:[2022-09-09 03:32:07,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-09 03:32:07,056] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default1]:[2022-09-09 03:32:07,049] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-09 03:32:07,049] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default7]:[2022-09-09 03:32:07,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-09 03:32:07,004] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default7]:[2022-09-09 03:32:07,040] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-09 03:32:07,040] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default5]:[2022-09-09 03:32:07,076] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-09 03:32:07,077] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default2]:[2022-09-09 03:32:07,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-09 03:32:07,043] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default5]:[2022-09-09 03:32:07,152] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-09 03:32:07,152] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default1]:[2022-09-09 03:32:07,166] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-09 03:32:07,166] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default5]:[2022-09-09 03:32:07,139] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-09 03:32:07,140] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default2]:[2022-09-09 03:32:07,212] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-09 03:32:07,212] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default3]:[2022-09-09 03:32:07,121] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-09 03:32:07,121] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default6]:[2022-09-09 03:32:07,204] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-09 03:32:07,204] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default3]:[2022-09-09 03:32:07,167] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-09 03:32:07,167] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default2]:[2022-09-09 03:32:07,175] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-09 03:32:07,175] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default0]:[2022-09-09 03:32:07,174] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-09 03:32:07,174] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default2]:[2022-09-09 03:32:07,256] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-09 03:32:07,257] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default7]:[2022-09-09 03:32:07,192] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-09 03:32:07,192] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default6]:[2022-09-09 03:32:07,194] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-09 03:32:07,194] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default4]:[2022-09-09 03:32:07,231] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-09 03:32:07,231] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default2]:[2022-09-09 03:32:07,330] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-09 03:32:07,330] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default7]:[2022-09-09 03:32:07,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-09 03:32:07,305] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default6]:[2022-09-09 03:32:07,300] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-09 03:32:07,301] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default2]:[2022-09-09 03:32:07,344] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-09 03:32:07,344] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default5]:[2022-09-09 03:32:07,349] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-09 03:32:07,349] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default0]:[2022-09-09 03:32:07,425] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-09 03:32:07,425] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default1]:[2022-09-09 03:32:07,427] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-09 03:32:07,427] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default6]:[2022-09-09 03:32:07,465] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-09 03:32:07,466] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default5]:[2022-09-09 03:32:07,403] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-09 03:32:07,403] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default3]:[2022-09-09 03:32:07,461] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-09 03:32:07,461] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default1]:[2022-09-09 03:32:07,523] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-09 03:32:07,523] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default2]:[2022-09-09 03:32:07,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-09 03:32:07,579] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default0]:[2022-09-09 03:32:07,541] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-09 03:32:07,541] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default0]:[2022-09-09 03:32:07,542] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-09 03:32:07,542] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default4]:[2022-09-09 03:32:07,566] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-09 03:32:07,567] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default6]:[2022-09-09 03:32:07,698] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-09 03:32:07,698] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default5]:[2022-09-09 03:32:07,741] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-09 03:32:07,741] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default4]:[2022-09-09 03:32:07,827] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-09 03:32:07,827] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default4]:[2022-09-09 03:32:07,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-09 03:32:07,889] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default3]:[2022-09-09 03:32:07,960] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-09 03:32:07,960] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default0]:[2022-09-09 03:32:07,892] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-09 03:32:07,892] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default7]:[2022-09-09 03:32:07,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-09 03:32:07,973] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default0]:[2022-09-09 03:32:08,019] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-09 03:32:08,019] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default3]:[2022-09-09 03:32:08,065] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-09 03:32:08,065] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default5]:[2022-09-09 03:32:08,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-09 03:32:08,046] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default4]:[2022-09-09 03:32:08,131] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-09 03:32:08,131] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default5]:[2022-09-09 03:32:08,177] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-09 03:32:08,177] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default7]:[2022-09-09 03:32:08,301] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-09 03:32:08,301] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default4]:[2022-09-09 03:32:08,253] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-09 03:32:08,253] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default2]:[2022-09-09 03:32:08,332] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-09 03:32:08,332] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default1]:[2022-09-09 03:32:08,452] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-09 03:32:08,452] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default3]:[2022-09-09 03:32:08,745] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-09 03:32:08,746] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default6]:[2022-09-09 03:32:08,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-09 03:32:08,808] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default7]:[2022-09-09 03:32:08,889] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-09 03:32:08,889] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default1]:[2022-09-09 03:32:08,982] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-09 03:32:08,982] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default3]:[2022-09-09 03:32:09,256] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-09 03:32:09,256] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default5]:[2022-09-09 03:32:09,395] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-09 03:32:09,395] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default5]:[2022-09-09 03:32:09,596] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-09 03:32:09,596] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default4]:[2022-09-09 03:32:09,639] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-09 03:32:09,639] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default0]:[2022-09-09 03:32:09,664] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-09 03:32:09,664] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default6]:[2022-09-09 03:32:09,966] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-09 03:32:09,966] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default2]:[2022-09-09 03:32:10,278] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-09 03:32:10,278] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default1]:[2022-09-09 03:32:10,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-09 03:32:10,335] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default2]:[2022-09-09 03:32:10,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-09 03:32:10,783] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default4]:[2022-09-09 03:32:11,026] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-09 03:32:11,026] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default3]:[2022-09-09 03:32:11,251] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-09 03:32:11,251] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default7]:[2022-09-09 03:32:11,340] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-09 03:32:11,340] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default7]:[2022-09-09 03:32:12,796] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-09 03:32:12,796] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default6]:[2022-09-09 03:32:12,841] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-09 03:32:12,841] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default3]:[2022-09-09 03:32:13,232] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-09 03:32:13,232] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default1]:[2022-09-09 03:32:13,339] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-09 03:32:13,339] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default0]:[2022-09-09 03:32:13,361] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-09 03:32:13,361] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default5]:[2022-09-09 03:32:13,695] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-09 03:32:13,695] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default2]:[2022-09-09 03:32:14,743] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-09 03:32:14,744] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default3]:[2022-09-09 03:32:14,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-09 03:32:14,862] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default4]:[2022-09-09 03:32:14,998] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-09 03:32:14,998] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default3]:[2022-09-09 03:32:15,110] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-09 03:32:15,110] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default2]:[2022-09-09 03:32:15,195] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-09 03:32:15,195] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default5]:[2022-09-09 03:32:15,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-09 03:32:15,186] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default1]:[2022-09-09 03:32:15,840] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-09 03:32:15,840] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default0]:[2022-09-09 03:32:17,737] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-09 03:32:17,737] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default2]:[2022-09-09 03:32:18,666] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-09 03:32:18,666] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default0]:[2022-09-09 03:32:20,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-09 03:32:20,315] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default0]:[2022-09-09 03:32:20,467] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-09 03:32:20,467] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default1]:[2022-09-09 03:32:20,512] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-09 03:32:20,512] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default1]:[2022-09-09 03:32:20,721] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-09 03:32:20,722] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default7]:[2022-09-09 03:32:21,637] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-09 03:32:21,638] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default6]:[2022-09-09 03:32:21,629] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-09 03:32:21,629] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default5]:[2022-09-09 03:32:22,320] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-09 03:32:22,320] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default4]:[2022-09-09 03:32:22,327] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-09 03:32:22,327] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default7]:[2022-09-09 03:32:22,591] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-09 03:32:22,591] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default6]:[2022-09-09 03:32:23,111] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-09 03:32:23,111] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default4]:[2022-09-09 03:32:23,098] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-09 03:32:23,098] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default7]:[2022-09-09 03:32:23,205] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-09 03:32:23,205] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default5]:[2022-09-09 03:32:23,190] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-09 03:32:23,191] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:time (ms) | save-checkpoint: 34915.82 -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,575] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-09 03:32:23,575] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step747/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default2]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default5]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default1]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default3]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default4]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default6]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]: successfully saved checkpoint at iteration 747 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default0]:[2022-09-09 03:32:23,576] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step747 is ready now! -[default7]: iteration 748/ 3100 | consumed samples: 1531904 | consumed tokens: 3137339392 | elapsed time per iteration (s): 176.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.657395E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.599 | TFLOPs: 118.40 | -[default7]: iteration 749/ 3100 | consumed samples: 1533952 | consumed tokens: 3141533696 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.712852E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 750/ 3100 | consumed samples: 1536000 | consumed tokens: 3145728000 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.560678E-01 | grad norm: 0.475 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]:---------------------------------------------------------------------------------------------------------- -[default7]:validation_pretraining loss at iteration 750 | lm loss value: 2.350833E+00 | lm loss PPL: 1.049431E+01 | -[default7]:---------------------------------------------------------------------------------------------------------- -[default7]: iteration 751/ 3100 | consumed samples: 1538048 | consumed tokens: 3149922304 | elapsed time per iteration (s): 183.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.564245E-01 | grad norm: 0.484 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.132 | TFLOPs: 113.65 | -[default7]: iteration 752/ 3100 | consumed samples: 1540096 | consumed tokens: 3154116608 | elapsed time per iteration (s): 140.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.700560E-01 | grad norm: 0.438 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.584 | TFLOPs: 148.88 | -[default7]: iteration 753/ 3100 | consumed samples: 1542144 | consumed tokens: 3158310912 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.535127E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.45 | -[default7]: iteration 754/ 3100 | consumed samples: 1544192 | consumed tokens: 3162505216 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.477547E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 755/ 3100 | consumed samples: 1546240 | consumed tokens: 3166699520 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.528105E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 756/ 3100 | consumed samples: 1548288 | consumed tokens: 3170893824 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.592971E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 757/ 3100 | consumed samples: 1550336 | consumed tokens: 3175088128 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.551129E-01 | grad norm: 0.448 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 758/ 3100 | consumed samples: 1552384 | consumed tokens: 3179282432 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.507421E-01 | grad norm: 0.430 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 759/ 3100 | consumed samples: 1554432 | consumed tokens: 3183476736 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.408598E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 760/ 3100 | consumed samples: 1556480 | consumed tokens: 3187671040 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.442482E-01 | grad norm: 0.425 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.78 | -[default7]: iteration 761/ 3100 | consumed samples: 1558528 | consumed tokens: 3191865344 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.539015E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 762/ 3100 | consumed samples: 1560576 | consumed tokens: 3196059648 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.555707E-01 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 763/ 3100 | consumed samples: 1562624 | consumed tokens: 3200253952 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.647161E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 764/ 3100 | consumed samples: 1564672 | consumed tokens: 3204448256 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.544710E-01 | grad norm: 0.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 765/ 3100 | consumed samples: 1566720 | consumed tokens: 3208642560 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.470440E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 766/ 3100 | consumed samples: 1568768 | consumed tokens: 3212836864 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.521234E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 767/ 3100 | consumed samples: 1570816 | consumed tokens: 3217031168 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.369622E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.49 | -[default7]: iteration 768/ 3100 | consumed samples: 1572864 | consumed tokens: 3221225472 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.396939E-01 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 769/ 3100 | consumed samples: 1574912 | consumed tokens: 3225419776 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.509664E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 770/ 3100 | consumed samples: 1576960 | consumed tokens: 3229614080 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.412591E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 771/ 3100 | consumed samples: 1579008 | consumed tokens: 3233808384 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.349345E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.84 | -[default7]: iteration 772/ 3100 | consumed samples: 1581056 | consumed tokens: 3238002688 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.589438E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 773/ 3100 | consumed samples: 1583104 | consumed tokens: 3242196992 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.390597E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.582 | TFLOPs: 148.86 | -[default7]: iteration 774/ 3100 | consumed samples: 1585152 | consumed tokens: 3246391296 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.431521E-01 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 775/ 3100 | consumed samples: 1587200 | consumed tokens: 3250585600 | elapsed time per iteration (s): 140.99 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.629166E-01 | grad norm: 0.464 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.526 | TFLOPs: 148.29 | -[default7]: iteration 776/ 3100 | consumed samples: 1589248 | consumed tokens: 3254779904 | elapsed time per iteration (s): 140.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.589482E-01 | grad norm: 0.543 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.567 | TFLOPs: 148.71 | -[default7]: iteration 777/ 3100 | consumed samples: 1591296 | consumed tokens: 3258974208 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.523413E-01 | grad norm: 0.457 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 778/ 3100 | consumed samples: 1593344 | consumed tokens: 3263168512 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.498532E-01 | grad norm: 0.531 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 779/ 3100 | consumed samples: 1595392 | consumed tokens: 3267362816 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.496006E-01 | grad norm: 0.474 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 780/ 3100 | consumed samples: 1597440 | consumed tokens: 3271557120 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.462717E-01 | grad norm: 0.457 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 781/ 3100 | consumed samples: 1599488 | consumed tokens: 3275751424 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.627107E-01 | grad norm: 0.431 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 782/ 3100 | consumed samples: 1601536 | consumed tokens: 3279945728 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.480740E-01 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 783/ 3100 | consumed samples: 1603584 | consumed tokens: 3284140032 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.608124E-01 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 784/ 3100 | consumed samples: 1605632 | consumed tokens: 3288334336 | elapsed time per iteration (s): 141.34 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.388847E-01 | grad norm: 0.467 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.490 | TFLOPs: 147.92 | -[default7]: iteration 785/ 3100 | consumed samples: 1607680 | consumed tokens: 3292528640 | elapsed time per iteration (s): 141.13 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.592289E-01 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.512 | TFLOPs: 148.14 | -[default7]: iteration 786/ 3100 | consumed samples: 1609728 | consumed tokens: 3296722944 | elapsed time per iteration (s): 141.26 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.424122E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.498 | TFLOPs: 148.00 | -[default7]: iteration 787/ 3100 | consumed samples: 1611776 | consumed tokens: 3300917248 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.559144E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 788/ 3100 | consumed samples: 1613824 | consumed tokens: 3305111552 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.509100E-01 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 789/ 3100 | consumed samples: 1615872 | consumed tokens: 3309305856 | elapsed time per iteration (s): 141.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.387939E-01 | grad norm: 0.730 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.525 | TFLOPs: 148.28 | -[default7]: iteration 790/ 3100 | consumed samples: 1617920 | consumed tokens: 3313500160 | elapsed time per iteration (s): 140.22 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.366747E-01 | grad norm: 0.413 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.606 | TFLOPs: 149.11 | -[default7]: iteration 791/ 3100 | consumed samples: 1619968 | consumed tokens: 3317694464 | elapsed time per iteration (s): 141.32 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.524020E-01 | grad norm: 0.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.492 | TFLOPs: 147.94 | -[default7]: iteration 792/ 3100 | consumed samples: 1622016 | consumed tokens: 3321888768 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.327796E-01 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 793/ 3100 | consumed samples: 1624064 | consumed tokens: 3326083072 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.470209E-01 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 794/ 3100 | consumed samples: 1626112 | consumed tokens: 3330277376 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.414188E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 795/ 3100 | consumed samples: 1628160 | consumed tokens: 3334471680 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.303939E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 796/ 3100 | consumed samples: 1630208 | consumed tokens: 3338665984 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.489515E-01 | grad norm: 0.439 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 797/ 3100 | consumed samples: 1632256 | consumed tokens: 3342860288 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.328991E-01 | grad norm: 0.467 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 798/ 3100 | consumed samples: 1634304 | consumed tokens: 3347054592 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.270795E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 799/ 3100 | consumed samples: 1636352 | consumed tokens: 3351248896 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.317913E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 800/ 3100 | consumed samples: 1638400 | consumed tokens: 3355443200 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.560950E-01 | grad norm: 0.821 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 801/ 3100 | consumed samples: 1640448 | consumed tokens: 3359637504 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.315621E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 802/ 3100 | consumed samples: 1642496 | consumed tokens: 3363831808 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.400911E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 803/ 3100 | consumed samples: 1644544 | consumed tokens: 3368026112 | elapsed time per iteration (s): 140.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.499773E-01 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.570 | TFLOPs: 148.73 | -[default7]: iteration 804/ 3100 | consumed samples: 1646592 | consumed tokens: 3372220416 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.424460E-01 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 805/ 3100 | consumed samples: 1648640 | consumed tokens: 3376414720 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.452367E-01 | grad norm: 0.430 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 806/ 3100 | consumed samples: 1650688 | consumed tokens: 3380609024 | elapsed time per iteration (s): 140.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.585927E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.531 | TFLOPs: 148.34 | -[default7]: iteration 807/ 3100 | consumed samples: 1652736 | consumed tokens: 3384803328 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.216262E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.499 | TFLOPs: 148.01 | -[default7]: iteration 808/ 3100 | consumed samples: 1654784 | consumed tokens: 3388997632 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.276610E-01 | grad norm: 0.454 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 809/ 3100 | consumed samples: 1656832 | consumed tokens: 3393191936 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.307635E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 810/ 3100 | consumed samples: 1658880 | consumed tokens: 3397386240 | elapsed time per iteration (s): 140.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.463262E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.533 | TFLOPs: 148.36 | -[default7]: iteration 811/ 3100 | consumed samples: 1660928 | consumed tokens: 3401580544 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.442689E-01 | grad norm: 0.446 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 812/ 3100 | consumed samples: 1662976 | consumed tokens: 3405774848 | elapsed time per iteration (s): 140.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.371426E-01 | grad norm: 0.413 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.587 | TFLOPs: 148.91 | -[default7]: iteration 813/ 3100 | consumed samples: 1665024 | consumed tokens: 3409969152 | elapsed time per iteration (s): 141.16 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.357400E-01 | grad norm: 0.413 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.508 | TFLOPs: 148.10 | -[default7]: iteration 814/ 3100 | consumed samples: 1667072 | consumed tokens: 3414163456 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.306884E-01 | grad norm: 0.425 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.80 | -[default7]: iteration 815/ 3100 | consumed samples: 1669120 | consumed tokens: 3418357760 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.396559E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 816/ 3100 | consumed samples: 1671168 | consumed tokens: 3422552064 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.388381E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 817/ 3100 | consumed samples: 1673216 | consumed tokens: 3426746368 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.558617E-01 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 818/ 3100 | consumed samples: 1675264 | consumed tokens: 3430940672 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.213256E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 819/ 3100 | consumed samples: 1677312 | consumed tokens: 3435134976 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.364251E-01 | grad norm: 0.455 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 820/ 3100 | consumed samples: 1679360 | consumed tokens: 3439329280 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.331456E-01 | grad norm: 0.526 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 821/ 3100 | consumed samples: 1681408 | consumed tokens: 3443523584 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.455363E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 822/ 3100 | consumed samples: 1683456 | consumed tokens: 3447717888 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.383476E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 823/ 3100 | consumed samples: 1685504 | consumed tokens: 3451912192 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.264795E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 824/ 3100 | consumed samples: 1687552 | consumed tokens: 3456106496 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.409384E-01 | grad norm: 0.437 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 825/ 3100 | consumed samples: 1689600 | consumed tokens: 3460300800 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.253796E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 826/ 3100 | consumed samples: 1691648 | consumed tokens: 3464495104 | elapsed time per iteration (s): 140.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.413131E-01 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.591 | TFLOPs: 148.95 | -[default7]: iteration 827/ 3100 | consumed samples: 1693696 | consumed tokens: 3468689408 | elapsed time per iteration (s): 141.06 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.426287E-01 | grad norm: 0.727 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.518 | TFLOPs: 148.21 | -[default7]: iteration 828/ 3100 | consumed samples: 1695744 | consumed tokens: 3472883712 | elapsed time per iteration (s): 140.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.345101E-01 | grad norm: 0.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.575 | TFLOPs: 148.78 | -[default7]: iteration 829/ 3100 | consumed samples: 1697792 | consumed tokens: 3477078016 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.348306E-01 | grad norm: 0.458 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 830/ 3100 | consumed samples: 1699840 | consumed tokens: 3481272320 | elapsed time per iteration (s): 140.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.483089E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.574 | TFLOPs: 148.78 | -[default7]: iteration 831/ 3100 | consumed samples: 1701888 | consumed tokens: 3485466624 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.349186E-01 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 832/ 3100 | consumed samples: 1703936 | consumed tokens: 3489660928 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.261941E-01 | grad norm: 0.446 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 833/ 3100 | consumed samples: 1705984 | consumed tokens: 3493855232 | elapsed time per iteration (s): 140.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.270610E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.555 | TFLOPs: 148.58 | -[default7]: iteration 834/ 3100 | consumed samples: 1708032 | consumed tokens: 3498049536 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.270074E-01 | grad norm: 0.488 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 835/ 3100 | consumed samples: 1710080 | consumed tokens: 3502243840 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.327900E-01 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 836/ 3100 | consumed samples: 1712128 | consumed tokens: 3506438144 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.216355E-01 | grad norm: 0.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 837/ 3100 | consumed samples: 1714176 | consumed tokens: 3510632448 | elapsed time per iteration (s): 141.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.350622E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.82 | -[default7]: iteration 838/ 3100 | consumed samples: 1716224 | consumed tokens: 3514826752 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.241619E-01 | grad norm: 0.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 839/ 3100 | consumed samples: 1718272 | consumed tokens: 3519021056 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.247711E-01 | grad norm: 0.885 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 840/ 3100 | consumed samples: 1720320 | consumed tokens: 3523215360 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.214081E-01 | grad norm: 0.517 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 841/ 3100 | consumed samples: 1722368 | consumed tokens: 3527409664 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.154350E-01 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 842/ 3100 | consumed samples: 1724416 | consumed tokens: 3531603968 | elapsed time per iteration (s): 141.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.348264E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.486 | TFLOPs: 147.88 | -[default7]: iteration 843/ 3100 | consumed samples: 1726464 | consumed tokens: 3535798272 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.365466E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 844/ 3100 | consumed samples: 1728512 | consumed tokens: 3539992576 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.225301E-01 | grad norm: 0.467 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 845/ 3100 | consumed samples: 1730560 | consumed tokens: 3544186880 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.198049E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 846/ 3100 | consumed samples: 1732608 | consumed tokens: 3548381184 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.331813E-01 | grad norm: 0.425 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 847/ 3100 | consumed samples: 1734656 | consumed tokens: 3552575488 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.267539E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 848/ 3100 | consumed samples: 1736704 | consumed tokens: 3556769792 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.246432E-01 | grad norm: 0.510 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 849/ 3100 | consumed samples: 1738752 | consumed tokens: 3560964096 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.269658E-01 | grad norm: 0.566 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.54 | -[default7]: iteration 850/ 3100 | consumed samples: 1740800 | consumed tokens: 3565158400 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.159718E-01 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 851/ 3100 | consumed samples: 1742848 | consumed tokens: 3569352704 | elapsed time per iteration (s): 141.09 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.331943E-01 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.516 | TFLOPs: 148.19 | -[default7]: iteration 852/ 3100 | consumed samples: 1744896 | consumed tokens: 3573547008 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.166435E-01 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.84 | -[default7]: iteration 853/ 3100 | consumed samples: 1746944 | consumed tokens: 3577741312 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.132509E-01 | grad norm: 0.455 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 854/ 3100 | consumed samples: 1748992 | consumed tokens: 3581935616 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.149827E-01 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 855/ 3100 | consumed samples: 1751040 | consumed tokens: 3586129920 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.216075E-01 | grad norm: 0.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 856/ 3100 | consumed samples: 1753088 | consumed tokens: 3590324224 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.050117E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 857/ 3100 | consumed samples: 1755136 | consumed tokens: 3594518528 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.251088E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 858/ 3100 | consumed samples: 1757184 | consumed tokens: 3598712832 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.165119E-01 | grad norm: 1.607 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 859/ 3100 | consumed samples: 1759232 | consumed tokens: 3602907136 | elapsed time per iteration (s): 140.23 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.384944E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.604 | TFLOPs: 149.09 | -[default7]: iteration 860/ 3100 | consumed samples: 1761280 | consumed tokens: 3607101440 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.348605E-01 | grad norm: 0.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 861/ 3100 | consumed samples: 1763328 | consumed tokens: 3611295744 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.217086E-01 | grad norm: 0.458 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 862/ 3100 | consumed samples: 1765376 | consumed tokens: 3615490048 | elapsed time per iteration (s): 140.14 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.195586E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.614 | TFLOPs: 149.18 | -[default7]: iteration 863/ 3100 | consumed samples: 1767424 | consumed tokens: 3619684352 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.053428E-01 | grad norm: 0.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 864/ 3100 | consumed samples: 1769472 | consumed tokens: 3623878656 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.290849E-01 | grad norm: 0.464 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 865/ 3100 | consumed samples: 1771520 | consumed tokens: 3628072960 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.157423E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 866/ 3100 | consumed samples: 1773568 | consumed tokens: 3632267264 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.302398E-01 | grad norm: 0.533 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.42 | -[default7]: iteration 867/ 3100 | consumed samples: 1775616 | consumed tokens: 3636461568 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.273674E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 868/ 3100 | consumed samples: 1777664 | consumed tokens: 3640655872 | elapsed time per iteration (s): 140.13 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.153969E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.615 | TFLOPs: 149.20 | -[default7]: iteration 869/ 3100 | consumed samples: 1779712 | consumed tokens: 3644850176 | elapsed time per iteration (s): 140.98 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.166309E-01 | grad norm: 0.469 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.527 | TFLOPs: 148.30 | -[default7]: iteration 870/ 3100 | consumed samples: 1781760 | consumed tokens: 3649044480 | elapsed time per iteration (s): 140.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.086902E-01 | grad norm: 0.498 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.573 | TFLOPs: 148.76 | -[default7]: iteration 871/ 3100 | consumed samples: 1783808 | consumed tokens: 3653238784 | elapsed time per iteration (s): 139.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.178931E-01 | grad norm: 0.437 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.657 | TFLOPs: 149.63 | -[default7]: iteration 872/ 3100 | consumed samples: 1785856 | consumed tokens: 3657433088 | elapsed time per iteration (s): 140.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.308790E-01 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.566 | TFLOPs: 148.69 | -[default7]: iteration 873/ 3100 | consumed samples: 1787904 | consumed tokens: 3661627392 | elapsed time per iteration (s): 140.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.295744E-01 | grad norm: 0.486 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.531 | TFLOPs: 148.34 | -[default7]: iteration 874/ 3100 | consumed samples: 1789952 | consumed tokens: 3665821696 | elapsed time per iteration (s): 140.20 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.200777E-01 | grad norm: 0.501 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.608 | TFLOPs: 149.13 | -[default7]: iteration 875/ 3100 | consumed samples: 1792000 | consumed tokens: 3670016000 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.255166E-01 | grad norm: 0.518 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 876/ 3100 | consumed samples: 1794048 | consumed tokens: 3674210304 | elapsed time per iteration (s): 140.11 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.194138E-01 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.617 | TFLOPs: 149.22 | -[default7]: iteration 877/ 3100 | consumed samples: 1796096 | consumed tokens: 3678404608 | elapsed time per iteration (s): 140.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.151216E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.573 | TFLOPs: 148.76 | -[default7]: iteration 878/ 3100 | consumed samples: 1798144 | consumed tokens: 3682598912 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.079217E-01 | grad norm: 0.438 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 879/ 3100 | consumed samples: 1800192 | consumed tokens: 3686793216 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.178559E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 880/ 3100 | consumed samples: 1802240 | consumed tokens: 3690987520 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.387575E-01 | grad norm: 0.436 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 881/ 3100 | consumed samples: 1804288 | consumed tokens: 3695181824 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.178268E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 882/ 3100 | consumed samples: 1806336 | consumed tokens: 3699376128 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.189129E-01 | grad norm: 0.446 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 883/ 3100 | consumed samples: 1808384 | consumed tokens: 3703570432 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.101585E-01 | grad norm: 0.452 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 884/ 3100 | consumed samples: 1810432 | consumed tokens: 3707764736 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.316416E-01 | grad norm: 0.597 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 885/ 3100 | consumed samples: 1812480 | consumed tokens: 3711959040 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.232807E-01 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 886/ 3100 | consumed samples: 1814528 | consumed tokens: 3716153344 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.144811E-01 | grad norm: 0.493 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 887/ 3100 | consumed samples: 1816576 | consumed tokens: 3720347648 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.253316E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 888/ 3100 | consumed samples: 1818624 | consumed tokens: 3724541952 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.229745E-01 | grad norm: 0.465 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 889/ 3100 | consumed samples: 1820672 | consumed tokens: 3728736256 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.135893E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 890/ 3100 | consumed samples: 1822720 | consumed tokens: 3732930560 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.307718E-01 | grad norm: 1.500 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 891/ 3100 | consumed samples: 1824768 | consumed tokens: 3737124864 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.021078E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 892/ 3100 | consumed samples: 1826816 | consumed tokens: 3741319168 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.140587E-01 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 893/ 3100 | consumed samples: 1828864 | consumed tokens: 3745513472 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.244199E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 894/ 3100 | consumed samples: 1830912 | consumed tokens: 3749707776 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.118487E-01 | grad norm: 0.434 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 895/ 3100 | consumed samples: 1832960 | consumed tokens: 3753902080 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.216431E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 896/ 3100 | consumed samples: 1835008 | consumed tokens: 3758096384 | elapsed time per iteration (s): 140.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.139313E-01 | grad norm: 0.472 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.570 | TFLOPs: 148.74 | -[default7]: iteration 897/ 3100 | consumed samples: 1837056 | consumed tokens: 3762290688 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.206877E-01 | grad norm: 0.425 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 898/ 3100 | consumed samples: 1839104 | consumed tokens: 3766484992 | elapsed time per iteration (s): 140.15 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.008033E-01 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.612 | TFLOPs: 149.17 | -[default7]: iteration 899/ 3100 | consumed samples: 1841152 | consumed tokens: 3770679296 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.095734E-01 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 900/ 3100 | consumed samples: 1843200 | consumed tokens: 3774873600 | elapsed time per iteration (s): 144.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.158326E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.183 | TFLOPs: 144.79 | -[default7]: iteration 901/ 3100 | consumed samples: 1845248 | consumed tokens: 3779067904 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.265692E-01 | grad norm: 0.437 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 902/ 3100 | consumed samples: 1847296 | consumed tokens: 3783262208 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.077681E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 903/ 3100 | consumed samples: 1849344 | consumed tokens: 3787456512 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.082355E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 904/ 3100 | consumed samples: 1851392 | consumed tokens: 3791650816 | elapsed time per iteration (s): 141.34 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.025584E-01 | grad norm: 0.449 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.490 | TFLOPs: 147.92 | -[default7]: iteration 905/ 3100 | consumed samples: 1853440 | consumed tokens: 3795845120 | elapsed time per iteration (s): 142.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.982311E-01 | grad norm: 0.437 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.423 | TFLOPs: 147.24 | -[default7]: iteration 906/ 3100 | consumed samples: 1855488 | consumed tokens: 3800039424 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.073849E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.76 | -[default7]: iteration 907/ 3100 | consumed samples: 1857536 | consumed tokens: 3804233728 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.213537E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 908/ 3100 | consumed samples: 1859584 | consumed tokens: 3808428032 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.029754E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 909/ 3100 | consumed samples: 1861632 | consumed tokens: 3812622336 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.031453E-01 | grad norm: 0.455 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 910/ 3100 | consumed samples: 1863680 | consumed tokens: 3816816640 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.120498E-01 | grad norm: 0.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 911/ 3100 | consumed samples: 1865728 | consumed tokens: 3821010944 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.192616E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 912/ 3100 | consumed samples: 1867776 | consumed tokens: 3825205248 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.138689E-01 | grad norm: 0.436 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 913/ 3100 | consumed samples: 1869824 | consumed tokens: 3829399552 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.029903E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 914/ 3100 | consumed samples: 1871872 | consumed tokens: 3833593856 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.098969E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 915/ 3100 | consumed samples: 1873920 | consumed tokens: 3837788160 | elapsed time per iteration (s): 140.06 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.978711E-01 | grad norm: 0.425 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.623 | TFLOPs: 149.27 | -[default7]: iteration 916/ 3100 | consumed samples: 1875968 | consumed tokens: 3841982464 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.192289E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 917/ 3100 | consumed samples: 1878016 | consumed tokens: 3846176768 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.992944E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 918/ 3100 | consumed samples: 1880064 | consumed tokens: 3850371072 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.020256E-01 | grad norm: 0.437 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.51 | -[default7]: iteration 919/ 3100 | consumed samples: 1882112 | consumed tokens: 3854565376 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.973640E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.84 | -[default7]: iteration 920/ 3100 | consumed samples: 1884160 | consumed tokens: 3858759680 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.121593E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 921/ 3100 | consumed samples: 1886208 | consumed tokens: 3862953984 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.128686E-01 | grad norm: 0.443 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 922/ 3100 | consumed samples: 1888256 | consumed tokens: 3867148288 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.083562E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 923/ 3100 | consumed samples: 1890304 | consumed tokens: 3871342592 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.065508E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 924/ 3100 | consumed samples: 1892352 | consumed tokens: 3875536896 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.039467E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 925/ 3100 | consumed samples: 1894400 | consumed tokens: 3879731200 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.978416E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 926/ 3100 | consumed samples: 1896448 | consumed tokens: 3883925504 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.039759E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 927/ 3100 | consumed samples: 1898496 | consumed tokens: 3888119808 | elapsed time per iteration (s): 140.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.985633E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.559 | TFLOPs: 148.62 | -[default7]: iteration 928/ 3100 | consumed samples: 1900544 | consumed tokens: 3892314112 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.110140E-01 | grad norm: 0.439 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 929/ 3100 | consumed samples: 1902592 | consumed tokens: 3896508416 | elapsed time per iteration (s): 140.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.152992E-01 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.562 | TFLOPs: 148.66 | -[default7]: iteration 930/ 3100 | consumed samples: 1904640 | consumed tokens: 3900702720 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.113844E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 931/ 3100 | consumed samples: 1906688 | consumed tokens: 3904897024 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.087582E-01 | grad norm: 0.430 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 932/ 3100 | consumed samples: 1908736 | consumed tokens: 3909091328 | elapsed time per iteration (s): 141.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.027556E-01 | grad norm: 0.434 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.525 | TFLOPs: 148.28 | -[default7]: iteration 933/ 3100 | consumed samples: 1910784 | consumed tokens: 3913285632 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.089884E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 934/ 3100 | consumed samples: 1912832 | consumed tokens: 3917479936 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.990896E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 935/ 3100 | consumed samples: 1914880 | consumed tokens: 3921674240 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.042574E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 936/ 3100 | consumed samples: 1916928 | consumed tokens: 3925868544 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.019127E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 937/ 3100 | consumed samples: 1918976 | consumed tokens: 3930062848 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.944468E-01 | grad norm: 0.413 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 938/ 3100 | consumed samples: 1921024 | consumed tokens: 3934257152 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.955564E-01 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 939/ 3100 | consumed samples: 1923072 | consumed tokens: 3938451456 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.197142E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 940/ 3100 | consumed samples: 1925120 | consumed tokens: 3942645760 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.063579E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 941/ 3100 | consumed samples: 1927168 | consumed tokens: 3946840064 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.975384E-01 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 942/ 3100 | consumed samples: 1929216 | consumed tokens: 3951034368 | elapsed time per iteration (s): 140.34 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.083259E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.593 | TFLOPs: 148.98 | -[default7]: iteration 943/ 3100 | consumed samples: 1931264 | consumed tokens: 3955228672 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.021673E-01 | grad norm: 0.448 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 944/ 3100 | consumed samples: 1933312 | consumed tokens: 3959422976 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.003049E-01 | grad norm: 0.423 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 945/ 3100 | consumed samples: 1935360 | consumed tokens: 3963617280 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.889974E-01 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 946/ 3100 | consumed samples: 1937408 | consumed tokens: 3967811584 | elapsed time per iteration (s): 140.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.061469E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.566 | TFLOPs: 148.69 | -[default7]: iteration 947/ 3100 | consumed samples: 1939456 | consumed tokens: 3972005888 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.062876E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 948/ 3100 | consumed samples: 1941504 | consumed tokens: 3976200192 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.968256E-01 | grad norm: 0.457 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 949/ 3100 | consumed samples: 1943552 | consumed tokens: 3980394496 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.920925E-01 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 950/ 3100 | consumed samples: 1945600 | consumed tokens: 3984588800 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.973413E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 951/ 3100 | consumed samples: 1947648 | consumed tokens: 3988783104 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.079307E-01 | grad norm: 0.472 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 952/ 3100 | consumed samples: 1949696 | consumed tokens: 3992977408 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.936596E-01 | grad norm: 0.426 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 953/ 3100 | consumed samples: 1951744 | consumed tokens: 3997171712 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.988407E-01 | grad norm: 0.439 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 954/ 3100 | consumed samples: 1953792 | consumed tokens: 4001366016 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.976789E-01 | grad norm: 0.515 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 955/ 3100 | consumed samples: 1955840 | consumed tokens: 4005560320 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.001088E-01 | grad norm: 0.474 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 956/ 3100 | consumed samples: 1957888 | consumed tokens: 4009754624 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.990865E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 957/ 3100 | consumed samples: 1959936 | consumed tokens: 4013948928 | elapsed time per iteration (s): 140.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.938019E-01 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.544 | TFLOPs: 148.47 | -[default7]: iteration 958/ 3100 | consumed samples: 1961984 | consumed tokens: 4018143232 | elapsed time per iteration (s): 142.38 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.109674E-01 | grad norm: 0.485 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.384 | TFLOPs: 146.84 | -[default7]: iteration 959/ 3100 | consumed samples: 1964032 | consumed tokens: 4022337536 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.020935E-01 | grad norm: 0.430 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 960/ 3100 | consumed samples: 1966080 | consumed tokens: 4026531840 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.956153E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 961/ 3100 | consumed samples: 1968128 | consumed tokens: 4030726144 | elapsed time per iteration (s): 141.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.916988E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.82 | -[default7]: iteration 962/ 3100 | consumed samples: 1970176 | consumed tokens: 4034920448 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.903318E-01 | grad norm: 0.430 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 963/ 3100 | consumed samples: 1972224 | consumed tokens: 4039114752 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.799146E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.54 | -[default7]: iteration 964/ 3100 | consumed samples: 1974272 | consumed tokens: 4043309056 | elapsed time per iteration (s): 141.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.936421E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.429 | TFLOPs: 147.30 | -[default7]: iteration 965/ 3100 | consumed samples: 1976320 | consumed tokens: 4047503360 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.942615E-01 | grad norm: 0.508 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 966/ 3100 | consumed samples: 1978368 | consumed tokens: 4051697664 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.098396E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 967/ 3100 | consumed samples: 1980416 | consumed tokens: 4055891968 | elapsed time per iteration (s): 140.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.056715E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.580 | TFLOPs: 148.84 | -[default7]: iteration 968/ 3100 | consumed samples: 1982464 | consumed tokens: 4060086272 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.888627E-01 | grad norm: 0.426 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 969/ 3100 | consumed samples: 1984512 | consumed tokens: 4064280576 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.968172E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 970/ 3100 | consumed samples: 1986560 | consumed tokens: 4068474880 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.026200E-01 | grad norm: 0.425 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 971/ 3100 | consumed samples: 1988608 | consumed tokens: 4072669184 | elapsed time per iteration (s): 141.04 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.013972E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.521 | TFLOPs: 148.23 | -[default7]: iteration 972/ 3100 | consumed samples: 1990656 | consumed tokens: 4076863488 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.944036E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 973/ 3100 | consumed samples: 1992704 | consumed tokens: 4081057792 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.927111E-01 | grad norm: 0.512 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 974/ 3100 | consumed samples: 1994752 | consumed tokens: 4085252096 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.964940E-01 | grad norm: 0.426 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 975/ 3100 | consumed samples: 1996800 | consumed tokens: 4089446400 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.993689E-01 | grad norm: 0.451 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.84 | -[default7]: iteration 976/ 3100 | consumed samples: 1998848 | consumed tokens: 4093640704 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.844869E-01 | grad norm: 0.447 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 977/ 3100 | consumed samples: 2000896 | consumed tokens: 4097835008 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.861656E-01 | grad norm: 0.467 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 978/ 3100 | consumed samples: 2002944 | consumed tokens: 4102029312 | elapsed time per iteration (s): 140.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.927233E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.565 | TFLOPs: 148.68 | -[default7]: iteration 979/ 3100 | consumed samples: 2004992 | consumed tokens: 4106223616 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.891068E-01 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 980/ 3100 | consumed samples: 2007040 | consumed tokens: 4110417920 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.964094E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 981/ 3100 | consumed samples: 2009088 | consumed tokens: 4114612224 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.931551E-01 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 982/ 3100 | consumed samples: 2011136 | consumed tokens: 4118806528 | elapsed time per iteration (s): 141.21 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.840465E-01 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.503 | TFLOPs: 148.06 | -[default7]: iteration 983/ 3100 | consumed samples: 2013184 | consumed tokens: 4123000832 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.961176E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 984/ 3100 | consumed samples: 2015232 | consumed tokens: 4127195136 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.094543E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 985/ 3100 | consumed samples: 2017280 | consumed tokens: 4131389440 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.728310E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 986/ 3100 | consumed samples: 2019328 | consumed tokens: 4135583744 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.837392E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 987/ 3100 | consumed samples: 2021376 | consumed tokens: 4139778048 | elapsed time per iteration (s): 140.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.900186E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.580 | TFLOPs: 148.84 | -[default7]: iteration 988/ 3100 | consumed samples: 2023424 | consumed tokens: 4143972352 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.988665E-01 | grad norm: 0.431 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 989/ 3100 | consumed samples: 2025472 | consumed tokens: 4148166656 | elapsed time per iteration (s): 143.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.900097E-01 | grad norm: 0.556 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.319 | TFLOPs: 146.18 | -[default7]: iteration 990/ 3100 | consumed samples: 2027520 | consumed tokens: 4152360960 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.877607E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 991/ 3100 | consumed samples: 2029568 | consumed tokens: 4156555264 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.729851E-01 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 992/ 3100 | consumed samples: 2031616 | consumed tokens: 4160749568 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.762800E-01 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 993/ 3100 | consumed samples: 2033664 | consumed tokens: 4164943872 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.776627E-01 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 994/ 3100 | consumed samples: 2035712 | consumed tokens: 4169138176 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.947433E-01 | grad norm: 0.551 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 995/ 3100 | consumed samples: 2037760 | consumed tokens: 4173332480 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.914293E-01 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.427 | TFLOPs: 147.28 | -[default0]:saving checkpoint at iteration 996 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default7]: iteration 996/ 3100 | consumed samples: 2039808 | consumed tokens: 4177526784 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.914959E-01 | grad norm: 0.529 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default0]:[2022-09-09 13:20:13,102] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step996 is begin to save! -[default0]:[2022-09-09 13:20:13,185] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_66-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_67-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,239] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_49-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,226] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_69-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,239] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_48-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,226] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_68-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_39-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_50-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_16-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_09-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_70-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_56-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_35-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_19-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_58-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_29-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_21-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_27-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_12-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_71-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_28-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_72-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_15-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_18-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_24-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_13-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_51-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_20-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_10-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,310] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_71_model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_55-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_42-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_25-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_43-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_44-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_32-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_61-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_57-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_38-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_33-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_14-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_05-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,319] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_01-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_52-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_47-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_34-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_53-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,394] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_64-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_04-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_11-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_46-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_03-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_59-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,394] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_36-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_54-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_60-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_45-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_08-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,397] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_63-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_26-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_17-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,394] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_07-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,407] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_37-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,394] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_31-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,407] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_41-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,408] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_40-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,394] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_65-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,394] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_23-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,394] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_22-model_00-model_states.pt... -[default4]:[2022-09-09 13:20:13,376] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_71_model_states.pt. -[default0]:[2022-09-09 13:20:13,394] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_06-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,399] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_62-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:13,394] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_30-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:16,697] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_72-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:16,698] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_74-model_00-model_states.pt... -[default0]:[2022-09-09 13:20:16,704] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_74-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:16,704] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_70_model_states.pt... -[default0]:[2022-09-09 13:20:16,706] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_70_model_states.pt. -[default4]:[2022-09-09 13:20:16,802] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_03-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:16,802] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_01_model_states.pt... -[default4]:[2022-09-09 13:20:16,807] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_01_model_states.pt. -[default0]:[2022-09-09 13:20:16,805] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_56-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:16,806] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_54_model_states.pt... -[default0]:[2022-09-09 13:20:16,814] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_54_model_states.pt. -[default4]:[2022-09-09 13:20:16,760] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_21-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:16,760] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_19_model_states.pt... -[default4]:[2022-09-09 13:20:16,765] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_19_model_states.pt. -[default4]:[2022-09-09 13:20:16,855] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_55-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:16,855] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_53_model_states.pt... -[default4]:[2022-09-09 13:20:16,869] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_53_model_states.pt. -[default0]:[2022-09-09 13:20:16,875] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_46-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:16,875] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_44_model_states.pt... -[default0]:[2022-09-09 13:20:16,882] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_44_model_states.pt. -[default0]:[2022-09-09 13:20:16,862] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_08-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:16,862] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_06_model_states.pt... -[default0]:[2022-09-09 13:20:16,866] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_06_model_states.pt. -[default0]:[2022-09-09 13:20:16,886] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_68-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:16,886] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_66_model_states.pt... -[default0]:[2022-09-09 13:20:16,894] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_66_model_states.pt. -[default4]:[2022-09-09 13:20:16,903] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_19-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:16,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_17_model_states.pt... -[default4]:[2022-09-09 13:20:16,911] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_17_model_states.pt. -[default4]:[2022-09-09 13:20:16,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_69-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:16,933] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_67_model_states.pt... -[default4]:[2022-09-09 13:20:16,937] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_67_model_states.pt. -[default4]:[2022-09-09 13:20:16,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_43-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:16,888] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_41_model_states.pt... -[default4]:[2022-09-09 13:20:16,894] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_41_model_states.pt. -[default0]:[2022-09-09 13:20:16,982] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_66-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:16,982] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_64_model_states.pt... -[default0]:[2022-09-09 13:20:16,995] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_64_model_states.pt. -[default4]:[2022-09-09 13:20:16,939] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_45-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:16,939] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_43_model_states.pt... -[default4]:[2022-09-09 13:20:16,949] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_43_model_states.pt. -[default4]:[2022-09-09 13:20:17,031] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_27-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,031] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_25_model_states.pt... -[default4]:[2022-09-09 13:20:17,068] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_15-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,069] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_13_model_states.pt... -[default0]:[2022-09-09 13:20:17,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_42-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,012] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_40_model_states.pt... -[default0]:[2022-09-09 13:20:17,015] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_40_model_states.pt. -[default0]:[2022-09-09 13:20:17,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_44-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_42_model_states.pt... -[default0]:[2022-09-09 13:20:17,062] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_42_model_states.pt. -[default4]:[2022-09-09 13:20:16,997] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_57-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:16,997] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_55_model_states.pt... -[default4]:[2022-09-09 13:20:17,007] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_55_model_states.pt. -[default0]:[2022-09-09 13:20:17,041] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_14-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,042] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_12_model_states.pt... -[default0]:[2022-09-09 13:20:17,059] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_12_model_states.pt. -[default4]:[2022-09-09 13:20:17,045] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_33-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_31_model_states.pt... -[default4]:[2022-09-09 13:20:17,057] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_31_model_states.pt. -[default0]:[2022-09-09 13:20:17,042] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_52-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,042] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_50_model_states.pt... -[default0]:[2022-09-09 13:20:17,057] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_50_model_states.pt. -[default4]:[2022-09-09 13:20:17,078] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_59-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,078] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_57_model_states.pt... -[default4]:[2022-09-09 13:20:17,084] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_57_model_states.pt. -[default0]:[2022-09-09 13:20:17,073] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_54-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,073] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_52_model_states.pt... -[default0]:[2022-09-09 13:20:17,078] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_52_model_states.pt. -[default4]:[2022-09-09 13:20:17,063] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_63-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,063] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_61_model_states.pt... -[default4]:[2022-09-09 13:20:17,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_61_model_states.pt. -[default0]:[2022-09-09 13:20:17,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_26-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,043] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_24_model_states.pt... -[default0]:[2022-09-09 13:20:17,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_24_model_states.pt. -[default4]:[2022-09-09 13:20:17,049] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_17-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,049] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_15_model_states.pt... -[default4]:[2022-09-09 13:20:17,059] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_15_model_states.pt. -[default4]:[2022-09-09 13:20:17,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_67-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,097] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_65_model_states.pt... -[default4]:[2022-09-09 13:20:17,099] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_65_model_states.pt. -[default4]:[2022-09-09 13:20:17,114] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_09-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,114] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_07_model_states.pt... -[default4]:[2022-09-09 13:20:17,117] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_07_model_states.pt. -[default0]:[2022-09-09 13:20:17,064] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_70-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,064] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_68_model_states.pt... -[default0]:[2022-09-09 13:20:17,081] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_68_model_states.pt. -[default4]:[2022-09-09 13:20:17,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_25_model_states.pt. -[default4]:[2022-09-09 13:20:17,069] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_31-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,069] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_29_model_states.pt... -[default4]:[2022-09-09 13:20:17,078] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_29_model_states.pt. -[default4]:[2022-09-09 13:20:17,067] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_71-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,067] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_69_model_states.pt... -[default4]:[2022-09-09 13:20:17,085] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_69_model_states.pt. -[default0]:[2022-09-09 13:20:17,137] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_40-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,138] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_38_model_states.pt... -[default0]:[2022-09-09 13:20:17,140] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_38_model_states.pt. -[default4]:[2022-09-09 13:20:17,074] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_13_model_states.pt. -[default0]:[2022-09-09 13:20:17,091] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_20-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,091] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_18_model_states.pt... -[default0]:[2022-09-09 13:20:17,093] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_18_model_states.pt. -[default4]:[2022-09-09 13:20:17,141] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_23-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,142] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_21_model_states.pt... -[default4]:[2022-09-09 13:20:17,146] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_21_model_states.pt. -[default0]:[2022-09-09 13:20:17,107] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_22-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,108] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_20_model_states.pt... -[default0]:[2022-09-09 13:20:17,110] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_20_model_states.pt. -[default0]:[2022-09-09 13:20:17,163] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_30-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,164] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_28_model_states.pt... -[default0]:[2022-09-09 13:20:17,166] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_28_model_states.pt. -[default4]:[2022-09-09 13:20:17,193] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_47-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,193] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_45_model_states.pt... -[default4]:[2022-09-09 13:20:17,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_45_model_states.pt. -[default4]:[2022-09-09 13:20:17,197] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_53-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,198] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_51_model_states.pt... -[default4]:[2022-09-09 13:20:17,202] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_51_model_states.pt. -[default0]:[2022-09-09 13:20:17,179] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_50-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,179] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_48_model_states.pt... -[default0]:[2022-09-09 13:20:17,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_48_model_states.pt. -[default0]:[2022-09-09 13:20:17,149] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_16-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,149] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_14_model_states.pt... -[default0]:[2022-09-09 13:20:17,151] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_14_model_states.pt. -[default4]:[2022-09-09 13:20:17,181] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_49-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,182] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_47_model_states.pt... -[default4]:[2022-09-09 13:20:17,193] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_47_model_states.pt. -[default4]:[2022-09-09 13:20:17,226] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_41-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,226] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_39_model_states.pt... -[default4]:[2022-09-09 13:20:17,236] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_39_model_states.pt. -[default0]:[2022-09-09 13:20:17,227] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_28-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_26_model_states.pt... -[default0]:[2022-09-09 13:20:17,237] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_26_model_states.pt. -[default0]:[2022-09-09 13:20:17,219] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_18-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,219] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_16_model_states.pt... -[default0]:[2022-09-09 13:20:17,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_16_model_states.pt. -[default4]:[2022-09-09 13:20:17,252] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_13-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,252] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_11_model_states.pt... -[default4]:[2022-09-09 13:20:17,264] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_11_model_states.pt. -[default0]:[2022-09-09 13:20:17,192] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_32-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,192] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_30_model_states.pt... -[default0]:[2022-09-09 13:20:17,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_30_model_states.pt. -[default0]:[2022-09-09 13:20:17,253] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_62-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,253] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_60_model_states.pt... -[default0]:[2022-09-09 13:20:17,257] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_60_model_states.pt. -[default4]:[2022-09-09 13:20:17,258] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_61-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,259] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_59_model_states.pt... -[default4]:[2022-09-09 13:20:17,275] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_59_model_states.pt. -[default4]:[2022-09-09 13:20:17,259] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_05-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,260] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_03_model_states.pt... -[default4]:[2022-09-09 13:20:17,269] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_03_model_states.pt. -[default0]:[2022-09-09 13:20:17,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_60-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,289] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_07-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,289] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_05_model_states.pt... -[default4]:[2022-09-09 13:20:17,296] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_05_model_states.pt. -[default0]:[2022-09-09 13:20:17,274] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_58-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,274] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_56_model_states.pt... -[default0]:[2022-09-09 13:20:17,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_56_model_states.pt. -[default4]:[2022-09-09 13:20:17,275] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_29-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,275] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_27_model_states.pt... -[default4]:[2022-09-09 13:20:17,277] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_27_model_states.pt. -[default0]:[2022-09-09 13:20:17,314] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_12-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,315] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_10_model_states.pt... -[default0]:[2022-09-09 13:20:17,317] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_10_model_states.pt. -[default0]:[2022-09-09 13:20:17,274] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_48-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,275] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_46_model_states.pt... -[default0]:[2022-09-09 13:20:17,277] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_46_model_states.pt. -[default0]:[2022-09-09 13:20:17,308] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_06-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,308] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_04_model_states.pt... -[default0]:[2022-09-09 13:20:17,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_04_model_states.pt. -[default0]:[2022-09-09 13:20:17,342] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_04-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,342] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_02_model_states.pt... -[default0]:[2022-09-09 13:20:17,345] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_02_model_states.pt. -[default0]:[2022-09-09 13:20:17,369] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_36-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,370] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_34_model_states.pt... -[default0]:[2022-09-09 13:20:17,372] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_34_model_states.pt. -[default0]:[2022-09-09 13:20:17,319] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_58_model_states.pt... -[default0]:[2022-09-09 13:20:17,321] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_58_model_states.pt. -[default4]:[2022-09-09 13:20:17,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_39-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,336] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_37_model_states.pt... -[default4]:[2022-09-09 13:20:17,344] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_37_model_states.pt. -[default4]:[2022-09-09 13:20:17,383] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_37-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,384] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_35_model_states.pt... -[default4]:[2022-09-09 13:20:17,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_35_model_states.pt. -[default4]:[2022-09-09 13:20:17,393] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_65-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,394] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_63_model_states.pt... -[default4]:[2022-09-09 13:20:17,396] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_63_model_states.pt. -[default0]:[2022-09-09 13:20:17,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_38-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_36_model_states.pt... -[default0]:[2022-09-09 13:20:17,419] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_36_model_states.pt. -[default0]:[2022-09-09 13:20:17,482] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_64-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,482] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_62_model_states.pt... -[default0]:[2022-09-09 13:20:17,484] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_62_model_states.pt. -[default4]:[2022-09-09 13:20:17,499] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_35-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,500] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_33_model_states.pt... -[default4]:[2022-09-09 13:20:17,506] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_33_model_states.pt. -[default0]:[2022-09-09 13:20:17,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_24-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,547] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_22_model_states.pt... -[default0]:[2022-09-09 13:20:17,554] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_22_model_states.pt. -[default0]:[2022-09-09 13:20:17,500] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_10-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,501] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_08_model_states.pt... -[default0]:[2022-09-09 13:20:17,506] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_08_model_states.pt. -[default4]:[2022-09-09 13:20:17,569] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_25-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,569] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_23_model_states.pt... -[default4]:[2022-09-09 13:20:17,571] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_23_model_states.pt. -[default0]:[2022-09-09 13:20:17,517] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_34-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:17,517] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_32_model_states.pt... -[default0]:[2022-09-09 13:20:17,519] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_32_model_states.pt. -[default4]:[2022-09-09 13:20:17,549] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_11-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,549] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_09_model_states.pt... -[default4]:[2022-09-09 13:20:17,551] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_09_model_states.pt. -[default4]:[2022-09-09 13:20:17,578] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_51-model_00-model_states.pt. -[default4]:[2022-09-09 13:20:17,578] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_49_model_states.pt... -[default4]:[2022-09-09 13:20:17,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_49_model_states.pt. -[default0]:[2022-09-09 13:20:18,328] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/layer_01-model_00-model_states.pt. -[default0]:[2022-09-09 13:20:18,328] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_00_model_states.pt -[default0]:[2022-09-09 13:20:18,328] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_00_model_states.pt... -[default0]:[2022-09-09 13:20:18,330] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/mp_rank_00_model_states.pt. -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default6]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default4]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default5]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default1]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default2]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default0]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default7]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default3]:[2022-09-09 13:20:18,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default4]:[2022-09-09 13:20:27,162] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-09 13:20:27,162] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default1]:[2022-09-09 13:20:27,208] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-09 13:20:27,208] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default2]:[2022-09-09 13:20:27,245] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-09 13:20:27,245] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default0]:[2022-09-09 13:20:27,434] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-09 13:20:27,434] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default6]:[2022-09-09 13:20:27,480] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-09 13:20:27,480] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default2]:[2022-09-09 13:20:27,589] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-09 13:20:27,590] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default0]:[2022-09-09 13:20:27,733] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-09 13:20:27,734] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default6]:[2022-09-09 13:20:27,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-09 13:20:27,728] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default3]:[2022-09-09 13:20:27,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-09 13:20:27,888] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default5]:[2022-09-09 13:20:27,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-09 13:20:27,888] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default2]:[2022-09-09 13:20:27,979] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-09 13:20:27,979] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default3]:[2022-09-09 13:20:27,967] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-09 13:20:27,968] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default2]:[2022-09-09 13:20:28,063] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-09 13:20:28,063] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default2]:[2022-09-09 13:20:27,983] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-09 13:20:27,984] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default4]:[2022-09-09 13:20:28,049] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-09 13:20:28,049] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default7]:[2022-09-09 13:20:28,006] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-09 13:20:28,006] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default4]:[2022-09-09 13:20:28,045] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-09 13:20:28,045] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default5]:[2022-09-09 13:20:28,133] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-09 13:20:28,133] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default6]:[2022-09-09 13:20:28,180] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-09 13:20:28,181] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default3]:[2022-09-09 13:20:28,196] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-09 13:20:28,196] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default6]:[2022-09-09 13:20:28,216] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-09 13:20:28,216] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default0]:[2022-09-09 13:20:28,312] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-09 13:20:28,312] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default3]:[2022-09-09 13:20:28,353] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-09 13:20:28,354] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default7]:[2022-09-09 13:20:28,356] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-09 13:20:28,356] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default1]:[2022-09-09 13:20:28,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-09 13:20:28,305] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default1]:[2022-09-09 13:20:28,345] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-09 13:20:28,346] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default0]:[2022-09-09 13:20:28,365] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-09 13:20:28,365] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default5]:[2022-09-09 13:20:28,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-09 13:20:28,385] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default1]:[2022-09-09 13:20:28,384] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-09 13:20:28,384] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default3]:[2022-09-09 13:20:28,348] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-09 13:20:28,348] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default0]:[2022-09-09 13:20:28,394] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-09 13:20:28,394] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default3]:[2022-09-09 13:20:28,384] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-09 13:20:28,384] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default2]:[2022-09-09 13:20:28,420] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-09 13:20:28,420] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default3]:[2022-09-09 13:20:28,439] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-09 13:20:28,439] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default0]:[2022-09-09 13:20:28,482] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-09 13:20:28,482] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default4]:[2022-09-09 13:20:28,626] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-09 13:20:28,627] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default6]:[2022-09-09 13:20:28,680] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-09 13:20:28,680] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default0]:[2022-09-09 13:20:28,693] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-09 13:20:28,693] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default1]:[2022-09-09 13:20:28,753] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-09 13:20:28,753] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default7]:[2022-09-09 13:20:28,772] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-09 13:20:28,773] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default5]:[2022-09-09 13:20:28,735] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-09 13:20:28,735] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default5]:[2022-09-09 13:20:28,840] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-09 13:20:28,840] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default2]:[2022-09-09 13:20:28,819] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-09 13:20:28,819] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default3]:[2022-09-09 13:20:28,894] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-09 13:20:28,894] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default2]:[2022-09-09 13:20:28,959] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-09 13:20:28,960] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default4]:[2022-09-09 13:20:29,022] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-09 13:20:29,022] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default1]:[2022-09-09 13:20:29,013] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-09 13:20:29,014] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default5]:[2022-09-09 13:20:28,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-09 13:20:28,994] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default1]:[2022-09-09 13:20:29,017] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-09 13:20:29,017] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default5]:[2022-09-09 13:20:29,204] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-09 13:20:29,204] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default7]:[2022-09-09 13:20:29,168] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-09 13:20:29,168] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default0]:[2022-09-09 13:20:29,237] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-09 13:20:29,237] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default3]:[2022-09-09 13:20:29,176] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-09 13:20:29,177] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default4]:[2022-09-09 13:20:29,216] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-09 13:20:29,216] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default0]:[2022-09-09 13:20:29,307] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-09 13:20:29,307] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default7]:[2022-09-09 13:20:29,236] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-09 13:20:29,236] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default7]:[2022-09-09 13:20:29,244] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-09 13:20:29,244] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default6]:[2022-09-09 13:20:29,312] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-09 13:20:29,313] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default5]:[2022-09-09 13:20:29,392] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-09 13:20:29,392] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default7]:[2022-09-09 13:20:29,404] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-09 13:20:29,404] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default4]:[2022-09-09 13:20:29,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-09 13:20:29,416] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default2]:[2022-09-09 13:20:29,471] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-09 13:20:29,471] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default2]:[2022-09-09 13:20:29,547] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-09 13:20:29,547] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default0]:[2022-09-09 13:20:29,586] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-09 13:20:29,587] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default6]:[2022-09-09 13:20:29,573] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-09 13:20:29,574] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default5]:[2022-09-09 13:20:29,613] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-09 13:20:29,613] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default6]:[2022-09-09 13:20:29,629] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-09 13:20:29,629] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default5]:[2022-09-09 13:20:29,788] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-09 13:20:29,788] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default2]:[2022-09-09 13:20:29,805] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-09 13:20:29,805] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default3]:[2022-09-09 13:20:29,750] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-09 13:20:29,750] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default0]:[2022-09-09 13:20:29,821] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-09 13:20:29,821] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default6]:[2022-09-09 13:20:29,774] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-09 13:20:29,774] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default6]:[2022-09-09 13:20:29,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-09 13:20:29,784] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default2]:[2022-09-09 13:20:29,822] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-09 13:20:29,822] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default4]:[2022-09-09 13:20:29,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-09 13:20:29,820] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default4]:[2022-09-09 13:20:29,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-09 13:20:29,831] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default3]:[2022-09-09 13:20:29,967] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-09 13:20:29,967] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default1]:[2022-09-09 13:20:29,998] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-09 13:20:29,999] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default3]:[2022-09-09 13:20:29,991] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-09 13:20:29,991] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default5]:[2022-09-09 13:20:29,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-09 13:20:29,993] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default5]:[2022-09-09 13:20:30,042] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-09 13:20:30,042] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default1]:[2022-09-09 13:20:30,052] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-09 13:20:30,052] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default3]:[2022-09-09 13:20:30,084] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-09 13:20:30,084] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default6]:[2022-09-09 13:20:30,080] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-09 13:20:30,080] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default4]:[2022-09-09 13:20:30,045] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-09 13:20:30,045] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default2]:[2022-09-09 13:20:30,013] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-09 13:20:30,013] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default7]:[2022-09-09 13:20:30,103] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-09 13:20:30,103] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default4]:[2022-09-09 13:20:30,134] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-09 13:20:30,134] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default2]:[2022-09-09 13:20:30,110] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-09 13:20:30,110] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default1]:[2022-09-09 13:20:30,168] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-09 13:20:30,168] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default2]:[2022-09-09 13:20:30,121] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-09 13:20:30,122] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default5]:[2022-09-09 13:20:30,145] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-09 13:20:30,145] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default6]:[2022-09-09 13:20:30,101] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-09 13:20:30,101] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default7]:[2022-09-09 13:20:30,118] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-09 13:20:30,118] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default7]:[2022-09-09 13:20:30,117] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-09 13:20:30,117] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default5]:[2022-09-09 13:20:30,134] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-09 13:20:30,134] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default4]:[2022-09-09 13:20:30,163] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-09 13:20:30,163] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default3]:[2022-09-09 13:20:30,230] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-09 13:20:30,230] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default5]:[2022-09-09 13:20:30,251] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-09 13:20:30,251] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default7]:[2022-09-09 13:20:30,248] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-09 13:20:30,248] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default3]:[2022-09-09 13:20:30,218] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-09 13:20:30,218] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default6]:[2022-09-09 13:20:30,184] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-09 13:20:30,184] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default6]:[2022-09-09 13:20:30,274] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-09 13:20:30,274] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default4]:[2022-09-09 13:20:30,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-09 13:20:30,262] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default3]:[2022-09-09 13:20:30,255] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-09 13:20:30,255] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default1]:[2022-09-09 13:20:30,325] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-09 13:20:30,325] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default4]:[2022-09-09 13:20:30,301] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-09 13:20:30,302] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default0]:[2022-09-09 13:20:30,327] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-09 13:20:30,327] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default2]:[2022-09-09 13:20:30,368] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-09 13:20:30,368] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default6]:[2022-09-09 13:20:30,367] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-09 13:20:30,367] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default7]:[2022-09-09 13:20:30,330] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-09 13:20:30,330] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default1]:[2022-09-09 13:20:30,354] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-09 13:20:30,354] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default7]:[2022-09-09 13:20:30,400] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-09 13:20:30,401] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default4]:[2022-09-09 13:20:30,453] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-09 13:20:30,453] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default1]:[2022-09-09 13:20:30,451] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-09 13:20:30,451] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default7]:[2022-09-09 13:20:30,451] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-09 13:20:30,452] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default7]:[2022-09-09 13:20:30,473] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-09 13:20:30,473] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default3]:[2022-09-09 13:20:30,465] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-09 13:20:30,465] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default0]:[2022-09-09 13:20:30,468] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-09 13:20:30,468] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default2]:[2022-09-09 13:20:30,450] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-09 13:20:30,450] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default2]:[2022-09-09 13:20:30,508] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-09 13:20:30,509] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default4]:[2022-09-09 13:20:30,541] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-09 13:20:30,541] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default0]:[2022-09-09 13:20:30,522] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-09 13:20:30,522] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default7]:[2022-09-09 13:20:30,529] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-09 13:20:30,529] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default0]:[2022-09-09 13:20:30,499] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-09 13:20:30,499] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default3]:[2022-09-09 13:20:30,537] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-09 13:20:30,538] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default1]:[2022-09-09 13:20:30,534] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-09 13:20:30,534] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default5]:[2022-09-09 13:20:30,544] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-09 13:20:30,544] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default1]:[2022-09-09 13:20:30,551] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-09 13:20:30,551] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default6]:[2022-09-09 13:20:30,567] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-09 13:20:30,567] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default7]:[2022-09-09 13:20:30,626] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-09 13:20:30,626] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default7]:[2022-09-09 13:20:30,671] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-09 13:20:30,671] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default1]:[2022-09-09 13:20:30,611] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-09 13:20:30,611] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default6]:[2022-09-09 13:20:30,664] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-09 13:20:30,665] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default5]:[2022-09-09 13:20:30,694] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-09 13:20:30,695] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default4]:[2022-09-09 13:20:30,693] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-09 13:20:30,693] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default5]:[2022-09-09 13:20:30,642] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-09 13:20:30,642] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default0]:[2022-09-09 13:20:30,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-09 13:20:30,670] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default0]:[2022-09-09 13:20:30,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-09 13:20:30,675] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default1]:[2022-09-09 13:20:30,666] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-09 13:20:30,666] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default2]:[2022-09-09 13:20:30,733] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-09 13:20:30,733] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default1]:[2022-09-09 13:20:30,774] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-09 13:20:30,774] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default7]:[2022-09-09 13:20:30,802] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-09 13:20:30,802] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default0]:[2022-09-09 13:20:30,773] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-09 13:20:30,773] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default3]:[2022-09-09 13:20:30,801] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-09 13:20:30,801] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default4]:[2022-09-09 13:20:30,804] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-09 13:20:30,804] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default3]:[2022-09-09 13:20:30,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-09 13:20:30,854] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default4]:[2022-09-09 13:20:30,851] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-09 13:20:30,851] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default3]:[2022-09-09 13:20:30,876] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-09 13:20:30,876] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default0]:[2022-09-09 13:20:30,811] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-09 13:20:30,811] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default2]:[2022-09-09 13:20:30,857] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-09 13:20:30,857] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default5]:[2022-09-09 13:20:30,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-09 13:20:30,854] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default7]:[2022-09-09 13:20:30,884] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-09 13:20:30,884] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default2]:[2022-09-09 13:20:30,894] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-09 13:20:30,894] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default1]:[2022-09-09 13:20:30,927] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-09 13:20:30,928] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default0]:[2022-09-09 13:20:31,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-09 13:20:31,023] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default4]:[2022-09-09 13:20:31,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-09 13:20:31,036] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default4]:[2022-09-09 13:20:31,042] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-09 13:20:31,042] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default3]:[2022-09-09 13:20:31,069] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-09 13:20:31,069] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default5]:[2022-09-09 13:20:31,058] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-09 13:20:31,058] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default0]:[2022-09-09 13:20:31,081] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-09 13:20:31,081] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default3]:[2022-09-09 13:20:31,114] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-09 13:20:31,114] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default6]:[2022-09-09 13:20:31,053] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-09 13:20:31,053] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default5]:[2022-09-09 13:20:31,104] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-09 13:20:31,104] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default5]:[2022-09-09 13:20:31,181] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-09 13:20:31,181] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default6]:[2022-09-09 13:20:31,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-09 13:20:31,186] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default5]:[2022-09-09 13:20:31,210] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-09 13:20:31,210] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default3]:[2022-09-09 13:20:31,187] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-09 13:20:31,188] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default7]:[2022-09-09 13:20:31,275] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-09 13:20:31,275] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default6]:[2022-09-09 13:20:31,225] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-09 13:20:31,225] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default5]:[2022-09-09 13:20:31,254] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-09 13:20:31,254] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default3]:[2022-09-09 13:20:31,281] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-09 13:20:31,281] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default1]:[2022-09-09 13:20:31,230] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-09 13:20:31,230] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default4]:[2022-09-09 13:20:31,242] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-09 13:20:31,242] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default7]:[2022-09-09 13:20:31,349] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-09 13:20:31,349] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default4]:[2022-09-09 13:20:31,323] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-09 13:20:31,323] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default2]:[2022-09-09 13:20:31,325] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-09 13:20:31,325] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default1]:[2022-09-09 13:20:31,286] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-09 13:20:31,286] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default4]:[2022-09-09 13:20:31,318] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-09 13:20:31,318] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default4]:[2022-09-09 13:20:31,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-09 13:20:31,417] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default0]:[2022-09-09 13:20:31,421] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-09 13:20:31,421] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default4]:[2022-09-09 13:20:31,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-09 13:20:31,416] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default4]:[2022-09-09 13:20:31,372] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-09 13:20:31,372] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default6]:[2022-09-09 13:20:31,390] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-09 13:20:31,390] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default2]:[2022-09-09 13:20:31,425] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-09 13:20:31,425] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default0]:[2022-09-09 13:20:31,479] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-09 13:20:31,479] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default0]:[2022-09-09 13:20:31,430] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-09 13:20:31,431] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default0]:[2022-09-09 13:20:31,444] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-09 13:20:31,444] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default1]:[2022-09-09 13:20:31,524] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-09 13:20:31,524] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default7]:[2022-09-09 13:20:31,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-09 13:20:31,502] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default4]:[2022-09-09 13:20:31,534] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-09 13:20:31,535] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default5]:[2022-09-09 13:20:31,526] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-09 13:20:31,526] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default1]:[2022-09-09 13:20:31,539] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-09 13:20:31,539] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default6]:[2022-09-09 13:20:31,614] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-09 13:20:31,614] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default2]:[2022-09-09 13:20:31,659] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-09 13:20:31,659] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default3]:[2022-09-09 13:20:31,573] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-09 13:20:31,574] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default7]:[2022-09-09 13:20:31,634] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-09 13:20:31,634] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default6]:[2022-09-09 13:20:31,639] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-09 13:20:31,639] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default6]:[2022-09-09 13:20:31,646] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-09 13:20:31,646] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default4]:[2022-09-09 13:20:31,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-09 13:20:31,636] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default7]:[2022-09-09 13:20:31,701] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-09 13:20:31,702] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default2]:[2022-09-09 13:20:31,707] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-09 13:20:31,707] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default6]:[2022-09-09 13:20:31,674] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-09 13:20:31,675] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default1]:[2022-09-09 13:20:31,839] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-09 13:20:31,839] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default2]:[2022-09-09 13:20:31,885] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-09 13:20:31,885] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default3]:[2022-09-09 13:20:31,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-09 13:20:31,910] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default6]:[2022-09-09 13:20:31,916] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-09 13:20:31,916] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default5]:[2022-09-09 13:20:32,119] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-09 13:20:32,119] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default1]:[2022-09-09 13:20:32,122] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-09 13:20:32,122] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default0]:[2022-09-09 13:20:32,074] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-09 13:20:32,074] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default6]:[2022-09-09 13:20:32,208] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-09 13:20:32,208] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default1]:[2022-09-09 13:20:32,252] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-09 13:20:32,252] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default1]:[2022-09-09 13:20:32,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-09 13:20:32,310] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default3]:[2022-09-09 13:20:32,296] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-09 13:20:32,296] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default2]:[2022-09-09 13:20:32,309] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-09 13:20:32,310] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default2]:[2022-09-09 13:20:32,323] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-09 13:20:32,323] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default7]:[2022-09-09 13:20:32,357] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-09 13:20:32,357] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default0]:[2022-09-09 13:20:32,406] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-09 13:20:32,406] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default5]:[2022-09-09 13:20:32,357] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-09 13:20:32,357] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default5]:[2022-09-09 13:20:32,558] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-09 13:20:32,558] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default2]:[2022-09-09 13:20:32,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-09 13:20:32,540] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default6]:[2022-09-09 13:20:32,510] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-09 13:20:32,510] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default1]:[2022-09-09 13:20:32,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-09 13:20:32,561] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default6]:[2022-09-09 13:20:32,588] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-09 13:20:32,588] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default1]:[2022-09-09 13:20:32,657] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-09 13:20:32,657] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default3]:[2022-09-09 13:20:32,601] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-09 13:20:32,601] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default5]:[2022-09-09 13:20:32,663] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-09 13:20:32,663] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default5]:[2022-09-09 13:20:32,610] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-09 13:20:32,611] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default3]:[2022-09-09 13:20:32,693] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-09 13:20:32,693] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default2]:[2022-09-09 13:20:32,764] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-09 13:20:32,764] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default7]:[2022-09-09 13:20:32,868] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-09 13:20:32,868] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default0]:[2022-09-09 13:20:32,898] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-09 13:20:32,898] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default7]:[2022-09-09 13:20:32,914] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-09 13:20:32,914] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default0]:[2022-09-09 13:20:32,901] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-09 13:20:32,901] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default5]:[2022-09-09 13:20:33,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-09 13:20:33,005] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default4]:[2022-09-09 13:20:33,104] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-09 13:20:33,104] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default6]:[2022-09-09 13:20:33,060] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-09 13:20:33,060] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default2]:[2022-09-09 13:20:33,179] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-09 13:20:33,179] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default7]:[2022-09-09 13:20:33,330] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-09 13:20:33,330] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default5]:[2022-09-09 13:20:33,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-09 13:20:33,385] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default3]:[2022-09-09 13:20:33,447] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-09 13:20:33,448] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default7]:[2022-09-09 13:20:33,498] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-09 13:20:33,498] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default4]:[2022-09-09 13:20:33,599] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-09 13:20:33,599] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default0]:[2022-09-09 13:20:33,684] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-09 13:20:33,685] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default7]:[2022-09-09 13:20:33,726] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-09 13:20:33,726] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default5]:[2022-09-09 13:20:33,724] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-09 13:20:33,724] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default7]:[2022-09-09 13:20:33,776] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-09 13:20:33,776] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default4]:[2022-09-09 13:20:33,964] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-09 13:20:33,964] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default4]:[2022-09-09 13:20:33,969] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-09 13:20:33,970] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default6]:[2022-09-09 13:20:34,131] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-09 13:20:34,131] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default1]:[2022-09-09 13:20:34,163] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-09 13:20:34,163] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default6]:[2022-09-09 13:20:34,274] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-09 13:20:34,274] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default0]:[2022-09-09 13:20:34,283] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-09 13:20:34,283] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default2]:[2022-09-09 13:20:34,431] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-09 13:20:34,431] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default3]:[2022-09-09 13:20:34,572] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-09 13:20:34,572] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default1]:[2022-09-09 13:20:34,730] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-09 13:20:34,730] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default2]:[2022-09-09 13:20:34,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-09 13:20:34,775] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default1]:[2022-09-09 13:20:34,826] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-09 13:20:34,826] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default3]:[2022-09-09 13:20:35,237] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-09 13:20:35,237] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default3]:[2022-09-09 13:20:35,322] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-09 13:20:35,322] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default0]:[2022-09-09 13:20:35,501] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-09 13:20:35,502] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default1]:[2022-09-09 13:20:35,978] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-09 13:20:35,979] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default2]:[2022-09-09 13:20:36,496] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-09 13:20:36,496] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default0]:[2022-09-09 13:20:36,625] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-09 13:20:36,625] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default3]:[2022-09-09 13:20:36,680] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-09 13:20:36,681] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default2]:[2022-09-09 13:20:37,301] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-09 13:20:37,301] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default1]:[2022-09-09 13:20:38,367] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-09 13:20:38,367] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default0]:[2022-09-09 13:20:38,356] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-09 13:20:38,356] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default4]:[2022-09-09 13:20:38,306] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-09 13:20:38,306] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default5]:[2022-09-09 13:20:38,490] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-09 13:20:38,490] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default7]:[2022-09-09 13:20:38,577] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-09 13:20:38,577] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default6]:[2022-09-09 13:20:38,567] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-09 13:20:38,567] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default7]:[2022-09-09 13:20:39,703] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-09 13:20:39,703] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default4]:[2022-09-09 13:20:39,793] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-09 13:20:39,793] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default2]:[2022-09-09 13:20:39,912] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-09 13:20:39,913] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default3]:[2022-09-09 13:20:39,912] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-09 13:20:39,912] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default5]:[2022-09-09 13:20:40,088] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-09 13:20:40,089] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default6]:[2022-09-09 13:20:40,329] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-09 13:20:40,329] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default1]:[2022-09-09 13:20:40,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-09 13:20:40,407] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default0]:[2022-09-09 13:20:40,449] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-09 13:20:40,449] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default1]:[2022-09-09 13:20:41,126] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-09 13:20:41,126] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default0]:[2022-09-09 13:20:41,266] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-09 13:20:41,271] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default5]:[2022-09-09 13:20:42,658] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-09 13:20:42,658] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default4]:[2022-09-09 13:20:42,928] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-09 13:20:42,928] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default6]:[2022-09-09 13:20:43,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-09 13:20:43,310] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default7]:[2022-09-09 13:20:43,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-09 13:20:43,313] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default7]:[2022-09-09 13:20:46,173] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-09 13:20:46,173] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,242] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-09 13:20:46,243] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step996/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default1]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]: successfully saved checkpoint at iteration 996 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default6]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default2]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:time (ms) | save-checkpoint: 33161.69 -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default0]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default4]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default3]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default5]:[2022-09-09 13:20:46,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step996 is ready now! -[default7]: iteration 997/ 3100 | consumed samples: 2041856 | consumed tokens: 4181721088 | elapsed time per iteration (s): 174.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.838151E-01 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.705 | TFLOPs: 119.49 | -[default7]: iteration 998/ 3100 | consumed samples: 2043904 | consumed tokens: 4185915392 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.865730E-01 | grad norm: 0.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.48 | -[default7]: iteration 999/ 3100 | consumed samples: 2045952 | consumed tokens: 4190109696 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.923684E-01 | grad norm: 0.487 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1000/ 3100 | consumed samples: 2048000 | consumed tokens: 4194304000 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.877558E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]:validation_pretraining loss at iteration 1000 | lm loss value: 2.367104E+00 | lm loss PPL: 1.066646E+01 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]: iteration 1001/ 3100 | consumed samples: 2050048 | consumed tokens: 4198498304 | elapsed time per iteration (s): 183.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.912402E-01 | grad norm: 0.565 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.153 | TFLOPs: 113.85 | -[default7]: iteration 1002/ 3100 | consumed samples: 2052096 | consumed tokens: 4202692608 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.816529E-01 | grad norm: 0.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1003/ 3100 | consumed samples: 2054144 | consumed tokens: 4206886912 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.862801E-01 | grad norm: 0.455 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1004/ 3100 | consumed samples: 2056192 | consumed tokens: 4211081216 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.889010E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1005/ 3100 | consumed samples: 2058240 | consumed tokens: 4215275520 | elapsed time per iteration (s): 141.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.965321E-01 | grad norm: 0.438 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.428 | TFLOPs: 147.28 | -[default7]: iteration 1006/ 3100 | consumed samples: 2060288 | consumed tokens: 4219469824 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.857387E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1007/ 3100 | consumed samples: 2062336 | consumed tokens: 4223664128 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.844967E-01 | grad norm: 0.521 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 1008/ 3100 | consumed samples: 2064384 | consumed tokens: 4227858432 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.965211E-01 | grad norm: 0.637 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1009/ 3100 | consumed samples: 2066432 | consumed tokens: 4232052736 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.073262E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1010/ 3100 | consumed samples: 2068480 | consumed tokens: 4236247040 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.791837E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 1011/ 3100 | consumed samples: 2070528 | consumed tokens: 4240441344 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.962272E-01 | grad norm: 0.473 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 1012/ 3100 | consumed samples: 2072576 | consumed tokens: 4244635648 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.947808E-01 | grad norm: 3.692 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1013/ 3100 | consumed samples: 2074624 | consumed tokens: 4248829952 | elapsed time per iteration (s): 141.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.946974E-01 | grad norm: 0.475 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.522 | TFLOPs: 148.25 | -[default7]: iteration 1014/ 3100 | consumed samples: 2076672 | consumed tokens: 4253024256 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.974309E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1015/ 3100 | consumed samples: 2078720 | consumed tokens: 4257218560 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.866861E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.82 | -[default7]: iteration 1016/ 3100 | consumed samples: 2080768 | consumed tokens: 4261412864 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.835230E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 1017/ 3100 | consumed samples: 2082816 | consumed tokens: 4265607168 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.874253E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 1018/ 3100 | consumed samples: 2084864 | consumed tokens: 4269801472 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.913274E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 1019/ 3100 | consumed samples: 2086912 | consumed tokens: 4273995776 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.974414E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 1020/ 3100 | consumed samples: 2088960 | consumed tokens: 4278190080 | elapsed time per iteration (s): 140.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.910810E-01 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.562 | TFLOPs: 148.66 | -[default7]: iteration 1021/ 3100 | consumed samples: 2091008 | consumed tokens: 4282384384 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.818021E-01 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1022/ 3100 | consumed samples: 2093056 | consumed tokens: 4286578688 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.923763E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1023/ 3100 | consumed samples: 2095104 | consumed tokens: 4290772992 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.887118E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1024/ 3100 | consumed samples: 2097152 | consumed tokens: 4294967296 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.745954E-01 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 1025/ 3100 | consumed samples: 2099200 | consumed tokens: 4299161600 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.781118E-01 | grad norm: 0.431 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.77 | -[default7]: iteration 1026/ 3100 | consumed samples: 2101248 | consumed tokens: 4303355904 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.006751E-01 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 1027/ 3100 | consumed samples: 2103296 | consumed tokens: 4307550208 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.729870E-01 | grad norm: 0.426 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 1028/ 3100 | consumed samples: 2105344 | consumed tokens: 4311744512 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.870713E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1029/ 3100 | consumed samples: 2107392 | consumed tokens: 4315938816 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.872476E-01 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 1030/ 3100 | consumed samples: 2109440 | consumed tokens: 4320133120 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.835925E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 1031/ 3100 | consumed samples: 2111488 | consumed tokens: 4324327424 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.877392E-01 | grad norm: 0.430 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.51 | -[default7]: iteration 1032/ 3100 | consumed samples: 2113536 | consumed tokens: 4328521728 | elapsed time per iteration (s): 140.35 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 9.010418E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.592 | TFLOPs: 148.96 | -[default7]: iteration 1033/ 3100 | consumed samples: 2115584 | consumed tokens: 4332716032 | elapsed time per iteration (s): 140.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.782333E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.563 | TFLOPs: 148.67 | -[default7]: iteration 1034/ 3100 | consumed samples: 2117632 | consumed tokens: 4336910336 | elapsed time per iteration (s): 141.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.817444E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.81 | -[default7]: iteration 1035/ 3100 | consumed samples: 2119680 | consumed tokens: 4341104640 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.805171E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1036/ 3100 | consumed samples: 2121728 | consumed tokens: 4345298944 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.641351E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1037/ 3100 | consumed samples: 2123776 | consumed tokens: 4349493248 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.837010E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1038/ 3100 | consumed samples: 2125824 | consumed tokens: 4353687552 | elapsed time per iteration (s): 142.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.681575E-01 | grad norm: 0.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.421 | TFLOPs: 147.21 | -[default7]: iteration 1039/ 3100 | consumed samples: 2127872 | consumed tokens: 4357881856 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.834780E-01 | grad norm: 0.470 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 1040/ 3100 | consumed samples: 2129920 | consumed tokens: 4362076160 | elapsed time per iteration (s): 141.18 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.808199E-01 | grad norm: 0.497 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.506 | TFLOPs: 148.09 | -[default7]: iteration 1041/ 3100 | consumed samples: 2131968 | consumed tokens: 4366270464 | elapsed time per iteration (s): 140.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.788642E-01 | grad norm: 0.732 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.576 | TFLOPs: 148.80 | -[default7]: iteration 1042/ 3100 | consumed samples: 2134016 | consumed tokens: 4370464768 | elapsed time per iteration (s): 140.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.745903E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.564 | TFLOPs: 148.67 | -[default7]: iteration 1043/ 3100 | consumed samples: 2136064 | consumed tokens: 4374659072 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.895258E-01 | grad norm: 0.453 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 1044/ 3100 | consumed samples: 2138112 | consumed tokens: 4378853376 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.703688E-01 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 1045/ 3100 | consumed samples: 2140160 | consumed tokens: 4383047680 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.739952E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1046/ 3100 | consumed samples: 2142208 | consumed tokens: 4387241984 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.799002E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.425 | TFLOPs: 147.26 | -[default7]: iteration 1047/ 3100 | consumed samples: 2144256 | consumed tokens: 4391436288 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.761346E-01 | grad norm: 0.436 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1048/ 3100 | consumed samples: 2146304 | consumed tokens: 4395630592 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.759588E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1049/ 3100 | consumed samples: 2148352 | consumed tokens: 4399824896 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.668680E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 1050/ 3100 | consumed samples: 2150400 | consumed tokens: 4404019200 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.850092E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 1051/ 3100 | consumed samples: 2152448 | consumed tokens: 4408213504 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.873193E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 1052/ 3100 | consumed samples: 2154496 | consumed tokens: 4412407808 | elapsed time per iteration (s): 142.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.799198E-01 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.345 | TFLOPs: 146.44 | -[default7]: iteration 1053/ 3100 | consumed samples: 2156544 | consumed tokens: 4416602112 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.605224E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1054/ 3100 | consumed samples: 2158592 | consumed tokens: 4420796416 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.802097E-01 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 1055/ 3100 | consumed samples: 2160640 | consumed tokens: 4424990720 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.793391E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 1056/ 3100 | consumed samples: 2162688 | consumed tokens: 4429185024 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.758123E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.46 | -[default7]: iteration 1057/ 3100 | consumed samples: 2164736 | consumed tokens: 4433379328 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.715823E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1058/ 3100 | consumed samples: 2166784 | consumed tokens: 4437573632 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.828679E-01 | grad norm: 0.436 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.581 | TFLOPs: 148.85 | -[default7]: iteration 1059/ 3100 | consumed samples: 2168832 | consumed tokens: 4441767936 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.643753E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1060/ 3100 | consumed samples: 2170880 | consumed tokens: 4445962240 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.722907E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.62 | -[default7]: iteration 1061/ 3100 | consumed samples: 2172928 | consumed tokens: 4450156544 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.721756E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 1062/ 3100 | consumed samples: 2174976 | consumed tokens: 4454350848 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.626798E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1063/ 3100 | consumed samples: 2177024 | consumed tokens: 4458545152 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.763951E-01 | grad norm: 0.681 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1064/ 3100 | consumed samples: 2179072 | consumed tokens: 4462739456 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.786749E-01 | grad norm: 3.196 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1065/ 3100 | consumed samples: 2181120 | consumed tokens: 4466933760 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.703948E-01 | grad norm: 43.498 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 1066/ 3100 | consumed samples: 2183168 | consumed tokens: 4471128064 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.907769E-01 | grad norm: 0.910 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 1067/ 3100 | consumed samples: 2185216 | consumed tokens: 4475322368 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.811554E-01 | grad norm: 0.485 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1068/ 3100 | consumed samples: 2187264 | consumed tokens: 4479516672 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.767113E-01 | grad norm: 0.434 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 1069/ 3100 | consumed samples: 2189312 | consumed tokens: 4483710976 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.698288E-01 | grad norm: 0.450 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.82 | -[default7]: iteration 1070/ 3100 | consumed samples: 2191360 | consumed tokens: 4487905280 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.752396E-01 | grad norm: 0.505 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 | -[default7]: iteration 1071/ 3100 | consumed samples: 2193408 | consumed tokens: 4492099584 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.894653E-01 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 1072/ 3100 | consumed samples: 2195456 | consumed tokens: 4496293888 | elapsed time per iteration (s): 140.27 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.717862E-01 | grad norm: 0.451 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.600 | TFLOPs: 149.05 | -[default7]: iteration 1073/ 3100 | consumed samples: 2197504 | consumed tokens: 4500488192 | elapsed time per iteration (s): 142.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.560552E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.421 | TFLOPs: 147.21 | -[default7]: iteration 1074/ 3100 | consumed samples: 2199552 | consumed tokens: 4504682496 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.734756E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1075/ 3100 | consumed samples: 2201600 | consumed tokens: 4508876800 | elapsed time per iteration (s): 141.13 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.751303E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.511 | TFLOPs: 148.14 | -[default7]: iteration 1076/ 3100 | consumed samples: 2203648 | consumed tokens: 4513071104 | elapsed time per iteration (s): 140.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.829372E-01 | grad norm: 0.434 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.602 | TFLOPs: 149.07 | -[default7]: iteration 1077/ 3100 | consumed samples: 2205696 | consumed tokens: 4517265408 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.615294E-01 | grad norm: 0.413 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1078/ 3100 | consumed samples: 2207744 | consumed tokens: 4521459712 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.836170E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 1079/ 3100 | consumed samples: 2209792 | consumed tokens: 4525654016 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.712075E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 1080/ 3100 | consumed samples: 2211840 | consumed tokens: 4529848320 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.763282E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1081/ 3100 | consumed samples: 2213888 | consumed tokens: 4534042624 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.705171E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 1082/ 3100 | consumed samples: 2215936 | consumed tokens: 4538236928 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.700835E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.75 | -[default7]: iteration 1083/ 3100 | consumed samples: 2217984 | consumed tokens: 4542431232 | elapsed time per iteration (s): 140.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.774021E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.552 | TFLOPs: 148.55 | -[default7]: iteration 1084/ 3100 | consumed samples: 2220032 | consumed tokens: 4546625536 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.697778E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1085/ 3100 | consumed samples: 2222080 | consumed tokens: 4550819840 | elapsed time per iteration (s): 140.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.722866E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.588 | TFLOPs: 148.92 | -[default7]: iteration 1086/ 3100 | consumed samples: 2224128 | consumed tokens: 4555014144 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.743532E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 1087/ 3100 | consumed samples: 2226176 | consumed tokens: 4559208448 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.803527E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 1088/ 3100 | consumed samples: 2228224 | consumed tokens: 4563402752 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.630428E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1089/ 3100 | consumed samples: 2230272 | consumed tokens: 4567597056 | elapsed time per iteration (s): 140.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.630934E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.551 | TFLOPs: 148.55 | -[default7]: iteration 1090/ 3100 | consumed samples: 2232320 | consumed tokens: 4571791360 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.719205E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 1091/ 3100 | consumed samples: 2234368 | consumed tokens: 4575985664 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.700457E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1092/ 3100 | consumed samples: 2236416 | consumed tokens: 4580179968 | elapsed time per iteration (s): 140.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.636131E-01 | grad norm: 0.467 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.558 | TFLOPs: 148.61 | -[default7]: iteration 1093/ 3100 | consumed samples: 2238464 | consumed tokens: 4584374272 | elapsed time per iteration (s): 140.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.677504E-01 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.532 | TFLOPs: 148.35 | -[default7]: iteration 1094/ 3100 | consumed samples: 2240512 | consumed tokens: 4588568576 | elapsed time per iteration (s): 142.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.764819E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.420 | TFLOPs: 147.21 | -[default7]: iteration 1095/ 3100 | consumed samples: 2242560 | consumed tokens: 4592762880 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.694722E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1096/ 3100 | consumed samples: 2244608 | consumed tokens: 4596957184 | elapsed time per iteration (s): 141.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.680462E-01 | grad norm: 0.474 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.523 | TFLOPs: 148.26 | -[default7]: iteration 1097/ 3100 | consumed samples: 2246656 | consumed tokens: 4601151488 | elapsed time per iteration (s): 141.27 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.657873E-01 | grad norm: 0.759 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.497 | TFLOPs: 147.99 | -[default7]: iteration 1098/ 3100 | consumed samples: 2248704 | consumed tokens: 4605345792 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.686389E-01 | grad norm: 0.423 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 1099/ 3100 | consumed samples: 2250752 | consumed tokens: 4609540096 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.669581E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 1100/ 3100 | consumed samples: 2252800 | consumed tokens: 4613734400 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.763332E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 1101/ 3100 | consumed samples: 2254848 | consumed tokens: 4617928704 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.701683E-01 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 1102/ 3100 | consumed samples: 2256896 | consumed tokens: 4622123008 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.618695E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 1103/ 3100 | consumed samples: 2258944 | consumed tokens: 4626317312 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.618756E-01 | grad norm: 0.445 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 1104/ 3100 | consumed samples: 2260992 | consumed tokens: 4630511616 | elapsed time per iteration (s): 140.30 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.809096E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.597 | TFLOPs: 149.02 | -[default7]: iteration 1105/ 3100 | consumed samples: 2263040 | consumed tokens: 4634705920 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.669677E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1106/ 3100 | consumed samples: 2265088 | consumed tokens: 4638900224 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.736670E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.72 | -[default7]: iteration 1107/ 3100 | consumed samples: 2267136 | consumed tokens: 4643094528 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.651536E-01 | grad norm: 0.467 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1108/ 3100 | consumed samples: 2269184 | consumed tokens: 4647288832 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.863736E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1109/ 3100 | consumed samples: 2271232 | consumed tokens: 4651483136 | elapsed time per iteration (s): 140.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.745772E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.594 | TFLOPs: 148.98 | -[default7]: iteration 1110/ 3100 | consumed samples: 2273280 | consumed tokens: 4655677440 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.653319E-01 | grad norm: 0.447 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 1111/ 3100 | consumed samples: 2275328 | consumed tokens: 4659871744 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.590649E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1112/ 3100 | consumed samples: 2277376 | consumed tokens: 4664066048 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.650764E-01 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 1113/ 3100 | consumed samples: 2279424 | consumed tokens: 4668260352 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.614132E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1114/ 3100 | consumed samples: 2281472 | consumed tokens: 4672454656 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.747631E-01 | grad norm: 0.434 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 1115/ 3100 | consumed samples: 2283520 | consumed tokens: 4676648960 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.678566E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 1116/ 3100 | consumed samples: 2285568 | consumed tokens: 4680843264 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.664088E-01 | grad norm: 0.485 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1117/ 3100 | consumed samples: 2287616 | consumed tokens: 4685037568 | elapsed time per iteration (s): 140.01 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.593987E-01 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.627 | TFLOPs: 149.32 | -[default7]: iteration 1118/ 3100 | consumed samples: 2289664 | consumed tokens: 4689231872 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.813714E-01 | grad norm: 0.487 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 1119/ 3100 | consumed samples: 2291712 | consumed tokens: 4693426176 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.554527E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1120/ 3100 | consumed samples: 2293760 | consumed tokens: 4697620480 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.713212E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 1121/ 3100 | consumed samples: 2295808 | consumed tokens: 4701814784 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.580009E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 1122/ 3100 | consumed samples: 2297856 | consumed tokens: 4706009088 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.647036E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1123/ 3100 | consumed samples: 2299904 | consumed tokens: 4710203392 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.485980E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1124/ 3100 | consumed samples: 2301952 | consumed tokens: 4714397696 | elapsed time per iteration (s): 140.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.498095E-01 | grad norm: 0.505 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.580 | TFLOPs: 148.84 | -[default7]: iteration 1125/ 3100 | consumed samples: 2304000 | consumed tokens: 4718592000 | elapsed time per iteration (s): 141.01 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.625084E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.524 | TFLOPs: 148.27 | -[default7]: iteration 1126/ 3100 | consumed samples: 2306048 | consumed tokens: 4722786304 | elapsed time per iteration (s): 140.38 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.644513E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.589 | TFLOPs: 148.93 | -[default7]: iteration 1127/ 3100 | consumed samples: 2308096 | consumed tokens: 4726980608 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.539450E-01 | grad norm: 0.480 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.47 | -[default7]: iteration 1128/ 3100 | consumed samples: 2310144 | consumed tokens: 4731174912 | elapsed time per iteration (s): 140.15 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.473229E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.613 | TFLOPs: 149.18 | -[default7]: iteration 1129/ 3100 | consumed samples: 2312192 | consumed tokens: 4735369216 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.542988E-01 | grad norm: 0.365 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1130/ 3100 | consumed samples: 2314240 | consumed tokens: 4739563520 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.512458E-01 | grad norm: 0.498 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 | -[default7]: iteration 1131/ 3100 | consumed samples: 2316288 | consumed tokens: 4743757824 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.590930E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 1132/ 3100 | consumed samples: 2318336 | consumed tokens: 4747952128 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.550988E-01 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1133/ 3100 | consumed samples: 2320384 | consumed tokens: 4752146432 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.486728E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1134/ 3100 | consumed samples: 2322432 | consumed tokens: 4756340736 | elapsed time per iteration (s): 141.04 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.444161E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.521 | TFLOPs: 148.23 | -[default7]: iteration 1135/ 3100 | consumed samples: 2324480 | consumed tokens: 4760535040 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.714760E-01 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 1136/ 3100 | consumed samples: 2326528 | consumed tokens: 4764729344 | elapsed time per iteration (s): 140.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.642254E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.557 | TFLOPs: 148.60 | -[default7]: iteration 1137/ 3100 | consumed samples: 2328576 | consumed tokens: 4768923648 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.654541E-01 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 1138/ 3100 | consumed samples: 2330624 | consumed tokens: 4773117952 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.563082E-01 | grad norm: 0.570 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1139/ 3100 | consumed samples: 2332672 | consumed tokens: 4777312256 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.541275E-01 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1140/ 3100 | consumed samples: 2334720 | consumed tokens: 4781506560 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.597182E-01 | grad norm: 0.457 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1141/ 3100 | consumed samples: 2336768 | consumed tokens: 4785700864 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.570809E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1142/ 3100 | consumed samples: 2338816 | consumed tokens: 4789895168 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.629782E-01 | grad norm: 0.464 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 1143/ 3100 | consumed samples: 2340864 | consumed tokens: 4794089472 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.644212E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1144/ 3100 | consumed samples: 2342912 | consumed tokens: 4798283776 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.501062E-01 | grad norm: 0.431 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 1145/ 3100 | consumed samples: 2344960 | consumed tokens: 4802478080 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.554754E-01 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 1146/ 3100 | consumed samples: 2347008 | consumed tokens: 4806672384 | elapsed time per iteration (s): 140.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.641993E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.538 | TFLOPs: 148.41 | -[default7]: iteration 1147/ 3100 | consumed samples: 2349056 | consumed tokens: 4810866688 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.488579E-01 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1148/ 3100 | consumed samples: 2351104 | consumed tokens: 4815060992 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.443226E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.83 | -[default7]: iteration 1149/ 3100 | consumed samples: 2353152 | consumed tokens: 4819255296 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.588728E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.72 | -[default7]: iteration 1150/ 3100 | consumed samples: 2355200 | consumed tokens: 4823449600 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.680663E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 1151/ 3100 | consumed samples: 2357248 | consumed tokens: 4827643904 | elapsed time per iteration (s): 141.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.496947E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.487 | TFLOPs: 147.89 | -[default7]: iteration 1152/ 3100 | consumed samples: 2359296 | consumed tokens: 4831838208 | elapsed time per iteration (s): 141.18 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.542156E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.506 | TFLOPs: 148.08 | -[default7]: iteration 1153/ 3100 | consumed samples: 2361344 | consumed tokens: 4836032512 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.510735E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 1154/ 3100 | consumed samples: 2363392 | consumed tokens: 4840226816 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.494881E-01 | grad norm: 0.748 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1155/ 3100 | consumed samples: 2365440 | consumed tokens: 4844421120 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.603202E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 | -[default7]: iteration 1156/ 3100 | consumed samples: 2367488 | consumed tokens: 4848615424 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.546921E-01 | grad norm: 0.413 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 1157/ 3100 | consumed samples: 2369536 | consumed tokens: 4852809728 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.529828E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1158/ 3100 | consumed samples: 2371584 | consumed tokens: 4857004032 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.491786E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 1159/ 3100 | consumed samples: 2373632 | consumed tokens: 4861198336 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.557901E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 1160/ 3100 | consumed samples: 2375680 | consumed tokens: 4865392640 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.620381E-01 | grad norm: 0.626 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 | -[default7]: iteration 1161/ 3100 | consumed samples: 2377728 | consumed tokens: 4869586944 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.607764E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 1162/ 3100 | consumed samples: 2379776 | consumed tokens: 4873781248 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.742535E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 1163/ 3100 | consumed samples: 2381824 | consumed tokens: 4877975552 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.487479E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 1164/ 3100 | consumed samples: 2383872 | consumed tokens: 4882169856 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.419985E-01 | grad norm: 0.627 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1165/ 3100 | consumed samples: 2385920 | consumed tokens: 4886364160 | elapsed time per iteration (s): 140.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.507009E-01 | grad norm: 0.477 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.552 | TFLOPs: 148.56 | -[default7]: iteration 1166/ 3100 | consumed samples: 2387968 | consumed tokens: 4890558464 | elapsed time per iteration (s): 141.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.383372E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.487 | TFLOPs: 147.89 | -[default7]: iteration 1167/ 3100 | consumed samples: 2390016 | consumed tokens: 4894752768 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.647926E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 1168/ 3100 | consumed samples: 2392064 | consumed tokens: 4898947072 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.456450E-01 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 1169/ 3100 | consumed samples: 2394112 | consumed tokens: 4903141376 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.738588E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1170/ 3100 | consumed samples: 2396160 | consumed tokens: 4907335680 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.627936E-01 | grad norm: 0.370 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1171/ 3100 | consumed samples: 2398208 | consumed tokens: 4911529984 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.529096E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 1172/ 3100 | consumed samples: 2400256 | consumed tokens: 4915724288 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.543205E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1173/ 3100 | consumed samples: 2402304 | consumed tokens: 4919918592 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.596967E-01 | grad norm: 0.453 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1174/ 3100 | consumed samples: 2404352 | consumed tokens: 4924112896 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.497063E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 1175/ 3100 | consumed samples: 2406400 | consumed tokens: 4928307200 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.540761E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 1176/ 3100 | consumed samples: 2408448 | consumed tokens: 4932501504 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.516973E-01 | grad norm: 0.557 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 1177/ 3100 | consumed samples: 2410496 | consumed tokens: 4936695808 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.527490E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 1178/ 3100 | consumed samples: 2412544 | consumed tokens: 4940890112 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.532165E-01 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 1179/ 3100 | consumed samples: 2414592 | consumed tokens: 4945084416 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.542085E-01 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1180/ 3100 | consumed samples: 2416640 | consumed tokens: 4949278720 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.382901E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 1181/ 3100 | consumed samples: 2418688 | consumed tokens: 4953473024 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.562599E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 1182/ 3100 | consumed samples: 2420736 | consumed tokens: 4957667328 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.527074E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 1183/ 3100 | consumed samples: 2422784 | consumed tokens: 4961861632 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.525259E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 1184/ 3100 | consumed samples: 2424832 | consumed tokens: 4966055936 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.395303E-01 | grad norm: 0.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.47 | -[default7]: iteration 1185/ 3100 | consumed samples: 2426880 | consumed tokens: 4970250240 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.588382E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1186/ 3100 | consumed samples: 2428928 | consumed tokens: 4974444544 | elapsed time per iteration (s): 141.14 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.532121E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.510 | TFLOPs: 148.13 | -[default7]: iteration 1187/ 3100 | consumed samples: 2430976 | consumed tokens: 4978638848 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.326402E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 1188/ 3100 | consumed samples: 2433024 | consumed tokens: 4982833152 | elapsed time per iteration (s): 141.38 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.657349E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.486 | TFLOPs: 147.88 | -[default7]: iteration 1189/ 3100 | consumed samples: 2435072 | consumed tokens: 4987027456 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.423672E-01 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 1190/ 3100 | consumed samples: 2437120 | consumed tokens: 4991221760 | elapsed time per iteration (s): 141.32 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.460560E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.491 | TFLOPs: 147.94 | -[default7]: iteration 1191/ 3100 | consumed samples: 2439168 | consumed tokens: 4995416064 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.491417E-01 | grad norm: 0.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 1192/ 3100 | consumed samples: 2441216 | consumed tokens: 4999610368 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.726321E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 1193/ 3100 | consumed samples: 2443264 | consumed tokens: 5003804672 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.412652E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1194/ 3100 | consumed samples: 2445312 | consumed tokens: 5007998976 | elapsed time per iteration (s): 140.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.456889E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.540 | TFLOPs: 148.43 | -[default7]: iteration 1195/ 3100 | consumed samples: 2447360 | consumed tokens: 5012193280 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.365040E-01 | grad norm: 0.600 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 1196/ 3100 | consumed samples: 2449408 | consumed tokens: 5016387584 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.387280E-01 | grad norm: 0.547 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 1197/ 3100 | consumed samples: 2451456 | consumed tokens: 5020581888 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.595588E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1198/ 3100 | consumed samples: 2453504 | consumed tokens: 5024776192 | elapsed time per iteration (s): 139.99 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.402281E-01 | grad norm: 1.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.630 | TFLOPs: 149.35 | -[default7]: iteration 1199/ 3100 | consumed samples: 2455552 | consumed tokens: 5028970496 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.545109E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 1200/ 3100 | consumed samples: 2457600 | consumed tokens: 5033164800 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.436703E-01 | grad norm: 0.533 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.582 | TFLOPs: 148.86 | -[default7]: iteration 1201/ 3100 | consumed samples: 2459648 | consumed tokens: 5037359104 | elapsed time per iteration (s): 141.08 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.503309E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.517 | TFLOPs: 148.19 | -[default7]: iteration 1202/ 3100 | consumed samples: 2461696 | consumed tokens: 5041553408 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.656533E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 1203/ 3100 | consumed samples: 2463744 | consumed tokens: 5045747712 | elapsed time per iteration (s): 140.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.637881E-01 | grad norm: 11.019 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.573 | TFLOPs: 148.77 | -[default7]: iteration 1204/ 3100 | consumed samples: 2465792 | consumed tokens: 5049942016 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.442508E-01 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 1205/ 3100 | consumed samples: 2467840 | consumed tokens: 5054136320 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.465989E-01 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 1206/ 3100 | consumed samples: 2469888 | consumed tokens: 5058330624 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.528880E-01 | grad norm: 0.483 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 1207/ 3100 | consumed samples: 2471936 | consumed tokens: 5062524928 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.544708E-01 | grad norm: 0.732 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.78 | -[default7]: iteration 1208/ 3100 | consumed samples: 2473984 | consumed tokens: 5066719232 | elapsed time per iteration (s): 140.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.419450E-01 | grad norm: 0.445 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.572 | TFLOPs: 148.75 | -[default7]: iteration 1209/ 3100 | consumed samples: 2476032 | consumed tokens: 5070913536 | elapsed time per iteration (s): 139.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.529807E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.643 | TFLOPs: 149.48 | -[default7]: iteration 1210/ 3100 | consumed samples: 2478080 | consumed tokens: 5075107840 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.591220E-01 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 1211/ 3100 | consumed samples: 2480128 | consumed tokens: 5079302144 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.593283E-01 | grad norm: 0.382 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 1212/ 3100 | consumed samples: 2482176 | consumed tokens: 5083496448 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.467790E-01 | grad norm: 1.223 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 1213/ 3100 | consumed samples: 2484224 | consumed tokens: 5087690752 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.429459E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 1214/ 3100 | consumed samples: 2486272 | consumed tokens: 5091885056 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.356001E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 1215/ 3100 | consumed samples: 2488320 | consumed tokens: 5096079360 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.553720E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1216/ 3100 | consumed samples: 2490368 | consumed tokens: 5100273664 | elapsed time per iteration (s): 140.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.428006E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.560 | TFLOPs: 148.63 | -[default7]: iteration 1217/ 3100 | consumed samples: 2492416 | consumed tokens: 5104467968 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.660216E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 1218/ 3100 | consumed samples: 2494464 | consumed tokens: 5108662272 | elapsed time per iteration (s): 140.29 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.566360E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.599 | TFLOPs: 149.03 | -[default7]: iteration 1219/ 3100 | consumed samples: 2496512 | consumed tokens: 5112856576 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.552861E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 1220/ 3100 | consumed samples: 2498560 | consumed tokens: 5117050880 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.488534E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 1221/ 3100 | consumed samples: 2500608 | consumed tokens: 5121245184 | elapsed time per iteration (s): 140.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.484948E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.529 | TFLOPs: 148.32 | -[default7]: iteration 1222/ 3100 | consumed samples: 2502656 | consumed tokens: 5125439488 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.482534E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 1223/ 3100 | consumed samples: 2504704 | consumed tokens: 5129633792 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.511000E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 1224/ 3100 | consumed samples: 2506752 | consumed tokens: 5133828096 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.432034E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1225/ 3100 | consumed samples: 2508800 | consumed tokens: 5138022400 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.489101E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 1226/ 3100 | consumed samples: 2510848 | consumed tokens: 5142216704 | elapsed time per iteration (s): 140.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.447630E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.549 | TFLOPs: 148.52 | -[default7]: iteration 1227/ 3100 | consumed samples: 2512896 | consumed tokens: 5146411008 | elapsed time per iteration (s): 140.32 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.407456E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.595 | TFLOPs: 148.99 | -[default7]: iteration 1228/ 3100 | consumed samples: 2514944 | consumed tokens: 5150605312 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.328148E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 1229/ 3100 | consumed samples: 2516992 | consumed tokens: 5154799616 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.368832E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 1230/ 3100 | consumed samples: 2519040 | consumed tokens: 5158993920 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.491234E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 1231/ 3100 | consumed samples: 2521088 | consumed tokens: 5163188224 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.546181E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1232/ 3100 | consumed samples: 2523136 | consumed tokens: 5167382528 | elapsed time per iteration (s): 140.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.421177E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.557 | TFLOPs: 148.60 | -[default7]: iteration 1233/ 3100 | consumed samples: 2525184 | consumed tokens: 5171576832 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.500627E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 1234/ 3100 | consumed samples: 2527232 | consumed tokens: 5175771136 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.429699E-01 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 1235/ 3100 | consumed samples: 2529280 | consumed tokens: 5179965440 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.319042E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 1236/ 3100 | consumed samples: 2531328 | consumed tokens: 5184159744 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.335946E-01 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.84 | -[default7]: iteration 1237/ 3100 | consumed samples: 2533376 | consumed tokens: 5188354048 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.411377E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1238/ 3100 | consumed samples: 2535424 | consumed tokens: 5192548352 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.312048E-01 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 1239/ 3100 | consumed samples: 2537472 | consumed tokens: 5196742656 | elapsed time per iteration (s): 140.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.299397E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.578 | TFLOPs: 148.81 | -[default7]: iteration 1240/ 3100 | consumed samples: 2539520 | consumed tokens: 5200936960 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.472600E-01 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 1241/ 3100 | consumed samples: 2541568 | consumed tokens: 5205131264 | elapsed time per iteration (s): 141.31 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.356061E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.493 | TFLOPs: 147.95 | -[default7]: iteration 1242/ 3100 | consumed samples: 2543616 | consumed tokens: 5209325568 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.489361E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 1243/ 3100 | consumed samples: 2545664 | consumed tokens: 5213519872 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.273150E-01 | grad norm: 0.413 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 1244/ 3100 | consumed samples: 2547712 | consumed tokens: 5217714176 | elapsed time per iteration (s): 141.11 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.420072E-01 | grad norm: 0.510 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.514 | TFLOPs: 148.16 | -[default7]: iteration 1245/ 3100 | consumed samples: 2549760 | consumed tokens: 5221908480 | elapsed time per iteration (s): 141.38 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.416669E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.486 | TFLOPs: 147.88 | -[default0]:saving checkpoint at iteration 1245 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-09 23:08:34,919] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1245 is begin to save! -[default4]:[2022-09-09 23:08:34,979] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_67-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_41-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_32-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_54-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_27-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_17-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_33-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_38-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_43-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_42-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_69-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_50-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,050] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_03-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_15-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_24-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:34,979] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_66-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_71-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_29-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_40-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_55-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_16-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_45-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_51-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,053] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_01-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_70-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_46-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_64-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_72-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_25-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_63-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,039] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_71_model_states.pt... -[default4]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_19-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_59-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_58-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_36-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_62-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_10-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_35-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_14-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_13-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_34-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_12-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_08-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_18-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_39-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_09-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_65-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_47-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_04-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,042] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_60-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_21-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_20-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_44-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_37-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_28-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_06-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_26-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_11-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_52-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_57-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_53-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,115] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_48-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,042] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_61-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,104] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_23-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,101] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_49-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_56-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,045] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_05-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_68-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_07-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,117] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_31-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,104] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_22-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:35,117] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_30-model_00-model_states.pt... -[default4]:[2022-09-09 23:08:35,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_71_model_states.pt. -[default0]:[2022-09-09 23:08:38,359] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_72-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,360] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_74-model_00-model_states.pt... -[default0]:[2022-09-09 23:08:38,366] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_74-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,367] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_70_model_states.pt... -[default0]:[2022-09-09 23:08:38,368] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_70_model_states.pt. -[default4]:[2022-09-09 23:08:38,415] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_43-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,415] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_41_model_states.pt... -[default4]:[2022-09-09 23:08:38,422] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_41_model_states.pt. -[default0]:[2022-09-09 23:08:38,550] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_42-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,551] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_40_model_states.pt... -[default0]:[2022-09-09 23:08:38,553] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_40_model_states.pt. -[default0]:[2022-09-09 23:08:38,524] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_14-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,525] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_12_model_states.pt... -[default0]:[2022-09-09 23:08:38,530] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_12_model_states.pt. -[default4]:[2022-09-09 23:08:38,646] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_33-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,647] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_31_model_states.pt... -[default4]:[2022-09-09 23:08:38,654] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_31_model_states.pt. -[default0]:[2022-09-09 23:08:38,657] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_22-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,658] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_20_model_states.pt... -[default0]:[2022-09-09 23:08:38,587] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_38-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,587] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_36_model_states.pt... -[default0]:[2022-09-09 23:08:38,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_36_model_states.pt. -[default0]:[2022-09-09 23:08:38,586] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_16-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,586] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_14_model_states.pt... -[default0]:[2022-09-09 23:08:38,598] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_14_model_states.pt. -[default0]:[2022-09-09 23:08:38,686] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_12-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,686] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_10_model_states.pt... -[default0]:[2022-09-09 23:08:38,694] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_10_model_states.pt. -[default4]:[2022-09-09 23:08:38,739] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_23-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,739] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_21_model_states.pt... -[default4]:[2022-09-09 23:08:38,694] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_49-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,695] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_47_model_states.pt... -[default4]:[2022-09-09 23:08:38,699] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_47_model_states.pt. -[default0]:[2022-09-09 23:08:38,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_20_model_states.pt. -[default0]:[2022-09-09 23:08:38,744] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_66-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,745] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_64_model_states.pt... -[default0]:[2022-09-09 23:08:38,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_64_model_states.pt. -[default4]:[2022-09-09 23:08:38,717] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_45-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,718] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_43_model_states.pt... -[default4]:[2022-09-09 23:08:38,727] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_43_model_states.pt. -[default4]:[2022-09-09 23:08:38,790] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_35-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,790] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_33_model_states.pt... -[default4]:[2022-09-09 23:08:38,798] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_33_model_states.pt. -[default4]:[2022-09-09 23:08:38,715] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_13-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,715] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_11_model_states.pt... -[default4]:[2022-09-09 23:08:38,717] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_11_model_states.pt. -[default0]:[2022-09-09 23:08:38,746] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_18-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,746] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_16_model_states.pt... -[default0]:[2022-09-09 23:08:38,769] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_16_model_states.pt. -[default4]:[2022-09-09 23:08:38,807] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_09-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,807] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_07_model_states.pt... -[default0]:[2022-09-09 23:08:38,737] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_44-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,738] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_42_model_states.pt... -[default0]:[2022-09-09 23:08:38,742] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_42_model_states.pt. -[default0]:[2022-09-09 23:08:38,780] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_48-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,780] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_46_model_states.pt... -[default0]:[2022-09-09 23:08:38,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_46_model_states.pt. -[default4]:[2022-09-09 23:08:38,768] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_57-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,768] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_55_model_states.pt... -[default4]:[2022-09-09 23:08:38,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_55_model_states.pt. -[default4]:[2022-09-09 23:08:38,749] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_21_model_states.pt. -[default4]:[2022-09-09 23:08:38,763] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_07-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,764] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_05_model_states.pt... -[default4]:[2022-09-09 23:08:38,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_05_model_states.pt. -[default4]:[2022-09-09 23:08:38,843] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_67-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,843] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_65_model_states.pt... -[default4]:[2022-09-09 23:08:38,846] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_65_model_states.pt. -[default0]:[2022-09-09 23:08:38,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_32-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,809] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_30_model_states.pt... -[default0]:[2022-09-09 23:08:38,814] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_30_model_states.pt. -[default4]:[2022-09-09 23:08:38,793] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_17-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,794] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_15_model_states.pt... -[default4]:[2022-09-09 23:08:38,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_15_model_states.pt. -[default4]:[2022-09-09 23:08:38,786] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_15-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,787] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_13_model_states.pt... -[default4]:[2022-09-09 23:08:38,789] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_13_model_states.pt. -[default4]:[2022-09-09 23:08:38,834] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_71-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_69_model_states.pt... -[default4]:[2022-09-09 23:08:38,842] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_69_model_states.pt. -[default4]:[2022-09-09 23:08:38,867] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_51-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,868] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_49_model_states.pt... -[default4]:[2022-09-09 23:08:38,875] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_49_model_states.pt. -[default0]:[2022-09-09 23:08:38,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_70-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_68_model_states.pt... -[default0]:[2022-09-09 23:08:38,869] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_68_model_states.pt. -[default4]:[2022-09-09 23:08:38,801] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_63-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,801] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_61_model_states.pt... -[default4]:[2022-09-09 23:08:38,806] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_61_model_states.pt. -[default4]:[2022-09-09 23:08:38,828] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_19-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,828] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_17_model_states.pt... -[default4]:[2022-09-09 23:08:38,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_17_model_states.pt. -[default0]:[2022-09-09 23:08:38,847] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_08-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_06_model_states.pt... -[default0]:[2022-09-09 23:08:38,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_06_model_states.pt. -[default4]:[2022-09-09 23:08:38,817] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_07_model_states.pt. -[default0]:[2022-09-09 23:08:38,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_04-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,889] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_02_model_states.pt... -[default0]:[2022-09-09 23:08:38,894] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_02_model_states.pt. -[default0]:[2022-09-09 23:08:38,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_60-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,856] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_58_model_states.pt... -[default0]:[2022-09-09 23:08:38,862] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_58_model_states.pt. -[default4]:[2022-09-09 23:08:38,921] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_21-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,921] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_19_model_states.pt... -[default0]:[2022-09-09 23:08:38,826] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_28-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,826] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_26_model_states.pt... -[default0]:[2022-09-09 23:08:38,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_26_model_states.pt. -[default0]:[2022-09-09 23:08:38,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_06-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,832] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_04_model_states.pt... -[default0]:[2022-09-09 23:08:38,834] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_04_model_states.pt. -[default0]:[2022-09-09 23:08:38,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_26-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_24_model_states.pt... -[default0]:[2022-09-09 23:08:38,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_24_model_states.pt. -[default0]:[2022-09-09 23:08:38,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_56-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,870] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_54_model_states.pt... -[default0]:[2022-09-09 23:08:38,874] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_54_model_states.pt. -[default4]:[2022-09-09 23:08:38,900] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_05-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,900] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_03_model_states.pt... -[default4]:[2022-09-09 23:08:38,902] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_03_model_states.pt. -[default0]:[2022-09-09 23:08:38,944] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_68-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,944] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_66_model_states.pt... -[default0]:[2022-09-09 23:08:38,885] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_50-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,885] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_48_model_states.pt... -[default0]:[2022-09-09 23:08:38,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_48_model_states.pt. -[default0]:[2022-09-09 23:08:38,978] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_40-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,971] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_62-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:38,971] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_60_model_states.pt... -[default0]:[2022-09-09 23:08:38,974] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_60_model_states.pt. -[default0]:[2022-09-09 23:08:39,003] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_34-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,003] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_32_model_states.pt... -[default0]:[2022-09-09 23:08:39,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_32_model_states.pt. -[default4]:[2022-09-09 23:08:38,913] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_39-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,913] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_37_model_states.pt... -[default4]:[2022-09-09 23:08:38,917] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_37_model_states.pt. -[default4]:[2022-09-09 23:08:38,926] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_19_model_states.pt. -[default4]:[2022-09-09 23:08:38,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_61-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,973] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_59_model_states.pt... -[default4]:[2022-09-09 23:08:38,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_59_model_states.pt. -[default0]:[2022-09-09 23:08:38,952] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_66_model_states.pt. -[default4]:[2022-09-09 23:08:39,029] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_31-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,030] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_29_model_states.pt... -[default4]:[2022-09-09 23:08:39,033] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_29_model_states.pt. -[default4]:[2022-09-09 23:08:39,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_41-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,056] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_39_model_states.pt... -[default4]:[2022-09-09 23:08:39,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_27-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,028] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_25_model_states.pt... -[default4]:[2022-09-09 23:08:39,033] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_25_model_states.pt. -[default4]:[2022-09-09 23:08:39,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_69-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,037] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_67_model_states.pt... -[default4]:[2022-09-09 23:08:39,039] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_67_model_states.pt. -[default4]:[2022-09-09 23:08:38,990] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_29-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:38,990] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_27_model_states.pt... -[default4]:[2022-09-09 23:08:38,996] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_27_model_states.pt. -[default0]:[2022-09-09 23:08:38,978] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_38_model_states.pt... -[default0]:[2022-09-09 23:08:38,996] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_38_model_states.pt. -[default4]:[2022-09-09 23:08:39,013] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_55-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,013] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_53_model_states.pt... -[default4]:[2022-09-09 23:08:39,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_53_model_states.pt. -[default0]:[2022-09-09 23:08:39,073] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_46-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,073] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_44_model_states.pt... -[default4]:[2022-09-09 23:08:39,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_25-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,028] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_23_model_states.pt... -[default4]:[2022-09-09 23:08:39,034] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_23_model_states.pt. -[default4]:[2022-09-09 23:08:39,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_65-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,070] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_63_model_states.pt... -[default4]:[2022-09-09 23:08:39,080] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_47-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,080] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_45_model_states.pt... -[default4]:[2022-09-09 23:08:39,116] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_45_model_states.pt. -[default0]:[2022-09-09 23:08:39,067] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_20-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,067] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_18_model_states.pt... -[default0]:[2022-09-09 23:08:39,069] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_18_model_states.pt. -[default0]:[2022-09-09 23:08:39,117] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_52-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,117] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_50_model_states.pt... -[default0]:[2022-09-09 23:08:39,129] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_50_model_states.pt. -[default4]:[2022-09-09 23:08:39,060] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_39_model_states.pt. -[default0]:[2022-09-09 23:08:39,119] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_54-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,120] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_52_model_states.pt... -[default0]:[2022-09-09 23:08:39,123] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_52_model_states.pt. -[default0]:[2022-09-09 23:08:39,109] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_30-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,110] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_28_model_states.pt... -[default0]:[2022-09-09 23:08:39,112] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_28_model_states.pt. -[default0]:[2022-09-09 23:08:39,105] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_24-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,105] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_22_model_states.pt... -[default0]:[2022-09-09 23:08:39,107] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_22_model_states.pt. -[default0]:[2022-09-09 23:08:39,118] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_44_model_states.pt. -[default4]:[2022-09-09 23:08:39,178] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_59-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,178] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_57_model_states.pt... -[default4]:[2022-09-09 23:08:39,185] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_57_model_states.pt. -[default0]:[2022-09-09 23:08:39,117] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_36-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,117] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_34_model_states.pt... -[default0]:[2022-09-09 23:08:39,127] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_34_model_states.pt. -[default4]:[2022-09-09 23:08:39,116] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_63_model_states.pt. -[default4]:[2022-09-09 23:08:39,205] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_37-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,205] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_35_model_states.pt... -[default4]:[2022-09-09 23:08:39,207] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_35_model_states.pt. -[default4]:[2022-09-09 23:08:39,191] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_53-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,192] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_51_model_states.pt... -[default4]:[2022-09-09 23:08:39,194] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_51_model_states.pt. -[default0]:[2022-09-09 23:08:39,187] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_64-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,187] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_62_model_states.pt... -[default0]:[2022-09-09 23:08:39,189] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_62_model_states.pt. -[default0]:[2022-09-09 23:08:39,248] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_58-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,248] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_56_model_states.pt... -[default0]:[2022-09-09 23:08:39,250] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_56_model_states.pt. -[default4]:[2022-09-09 23:08:39,282] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_11-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,283] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_09_model_states.pt... -[default4]:[2022-09-09 23:08:39,287] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_09_model_states.pt. -[default0]:[2022-09-09 23:08:39,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_10-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:39,305] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_08_model_states.pt... -[default0]:[2022-09-09 23:08:39,307] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_08_model_states.pt. -[default4]:[2022-09-09 23:08:39,667] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_03-model_00-model_states.pt. -[default4]:[2022-09-09 23:08:39,668] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_01_model_states.pt... -[default4]:[2022-09-09 23:08:39,671] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_01_model_states.pt. -[default0]:[2022-09-09 23:08:40,409] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/layer_01-model_00-model_states.pt. -[default0]:[2022-09-09 23:08:40,409] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_00_model_states.pt -[default0]:[2022-09-09 23:08:40,409] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_00_model_states.pt... -[default0]:[2022-09-09 23:08:40,411] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/mp_rank_00_model_states.pt. -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default0]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default0]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default0]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default4]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default4]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default4]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default4]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default4]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default0]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default4]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default4]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default4]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default0]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default0]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default5]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default1]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default5]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default2]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default2]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default6]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default6]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default0]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default3]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default1]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default7]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default4]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default7]:[2022-09-09 23:08:40,516] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default3]:[2022-09-09 23:08:40,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default0]:[2022-09-09 23:08:48,772] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-09 23:08:48,772] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default3]:[2022-09-09 23:08:49,127] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-09 23:08:49,127] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default5]:[2022-09-09 23:08:49,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-09 23:08:49,648] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default6]:[2022-09-09 23:08:49,694] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-09 23:08:49,695] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default4]:[2022-09-09 23:08:49,720] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-09 23:08:49,720] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default7]:[2022-09-09 23:08:49,927] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-09 23:08:49,928] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default5]:[2022-09-09 23:08:49,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-09 23:08:49,994] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default7]:[2022-09-09 23:08:50,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-09 23:08:50,028] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default2]:[2022-09-09 23:08:50,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-09 23:08:50,020] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default1]:[2022-09-09 23:08:50,119] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-09 23:08:50,119] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default0]:[2022-09-09 23:08:50,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-09 23:08:50,156] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default2]:[2022-09-09 23:08:50,238] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-09 23:08:50,239] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default1]:[2022-09-09 23:08:50,441] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-09 23:08:50,441] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default0]:[2022-09-09 23:08:50,452] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-09 23:08:50,452] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default2]:[2022-09-09 23:08:50,497] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-09 23:08:50,497] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default2]:[2022-09-09 23:08:50,496] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-09 23:08:50,496] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default3]:[2022-09-09 23:08:50,557] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-09 23:08:50,557] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default0]:[2022-09-09 23:08:50,555] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-09 23:08:50,555] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default5]:[2022-09-09 23:08:50,740] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-09 23:08:50,740] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default1]:[2022-09-09 23:08:50,835] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-09 23:08:50,835] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default3]:[2022-09-09 23:08:50,892] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-09 23:08:50,893] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default0]:[2022-09-09 23:08:50,900] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-09 23:08:50,901] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default4]:[2022-09-09 23:08:51,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-09 23:08:51,016] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default6]:[2022-09-09 23:08:51,060] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-09 23:08:51,060] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default1]:[2022-09-09 23:08:51,073] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-09 23:08:51,073] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default6]:[2022-09-09 23:08:51,171] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-09 23:08:51,171] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default7]:[2022-09-09 23:08:51,096] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-09 23:08:51,096] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default2]:[2022-09-09 23:08:51,164] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-09 23:08:51,164] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default3]:[2022-09-09 23:08:51,178] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-09 23:08:51,178] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default6]:[2022-09-09 23:08:51,288] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-09 23:08:51,288] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default1]:[2022-09-09 23:08:51,241] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-09 23:08:51,241] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default3]:[2022-09-09 23:08:51,333] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-09 23:08:51,334] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default7]:[2022-09-09 23:08:51,329] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-09 23:08:51,329] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default4]:[2022-09-09 23:08:51,399] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-09 23:08:51,399] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default7]:[2022-09-09 23:08:51,446] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-09 23:08:51,446] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default4]:[2022-09-09 23:08:51,420] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-09 23:08:51,420] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default6]:[2022-09-09 23:08:51,432] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-09 23:08:51,432] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default0]:[2022-09-09 23:08:51,512] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-09 23:08:51,512] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default4]:[2022-09-09 23:08:51,460] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-09 23:08:51,460] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default7]:[2022-09-09 23:08:51,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-09 23:08:51,559] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default4]:[2022-09-09 23:08:51,533] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-09 23:08:51,533] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default7]:[2022-09-09 23:08:51,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-09 23:08:51,560] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default1]:[2022-09-09 23:08:51,541] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-09 23:08:51,542] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default2]:[2022-09-09 23:08:51,603] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-09 23:08:51,603] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default6]:[2022-09-09 23:08:51,612] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-09 23:08:51,612] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default0]:[2022-09-09 23:08:51,607] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-09 23:08:51,607] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default5]:[2022-09-09 23:08:51,661] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-09 23:08:51,661] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default0]:[2022-09-09 23:08:51,590] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-09 23:08:51,590] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default4]:[2022-09-09 23:08:51,622] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-09 23:08:51,622] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default0]:[2022-09-09 23:08:51,691] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-09 23:08:51,691] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default0]:[2022-09-09 23:08:51,637] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-09 23:08:51,637] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default2]:[2022-09-09 23:08:51,637] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-09 23:08:51,637] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default5]:[2022-09-09 23:08:51,648] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-09 23:08:51,648] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default3]:[2022-09-09 23:08:51,752] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-09 23:08:51,752] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default2]:[2022-09-09 23:08:51,677] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-09 23:08:51,677] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default1]:[2022-09-09 23:08:51,717] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-09 23:08:51,717] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default7]:[2022-09-09 23:08:51,705] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-09 23:08:51,705] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default5]:[2022-09-09 23:08:51,711] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-09 23:08:51,711] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default5]:[2022-09-09 23:08:51,833] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-09 23:08:51,833] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default2]:[2022-09-09 23:08:51,816] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-09 23:08:51,816] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default3]:[2022-09-09 23:08:51,842] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-09 23:08:51,842] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default1]:[2022-09-09 23:08:51,799] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-09 23:08:51,799] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default4]:[2022-09-09 23:08:51,858] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-09 23:08:51,859] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default5]:[2022-09-09 23:08:51,878] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-09 23:08:51,879] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default3]:[2022-09-09 23:08:51,902] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-09 23:08:51,902] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default6]:[2022-09-09 23:08:51,814] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-09 23:08:51,814] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default6]:[2022-09-09 23:08:51,883] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-09 23:08:51,883] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default3]:[2022-09-09 23:08:51,913] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-09 23:08:51,913] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default4]:[2022-09-09 23:08:51,951] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-09 23:08:51,951] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default3]:[2022-09-09 23:08:51,922] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-09 23:08:51,922] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default4]:[2022-09-09 23:08:51,990] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-09 23:08:51,990] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default6]:[2022-09-09 23:08:52,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-09 23:08:52,037] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default6]:[2022-09-09 23:08:52,031] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-09 23:08:52,031] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default3]:[2022-09-09 23:08:52,098] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-09 23:08:52,098] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default2]:[2022-09-09 23:08:52,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-09 23:08:52,046] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default1]:[2022-09-09 23:08:52,093] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-09 23:08:52,093] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default1]:[2022-09-09 23:08:52,130] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-09 23:08:52,130] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default2]:[2022-09-09 23:08:52,117] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-09 23:08:52,117] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default7]:[2022-09-09 23:08:52,104] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-09 23:08:52,104] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default2]:[2022-09-09 23:08:52,109] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-09 23:08:52,109] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default0]:[2022-09-09 23:08:52,146] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-09 23:08:52,146] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default1]:[2022-09-09 23:08:52,202] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-09 23:08:52,202] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default0]:[2022-09-09 23:08:52,233] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-09 23:08:52,233] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default1]:[2022-09-09 23:08:52,208] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-09 23:08:52,208] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default2]:[2022-09-09 23:08:52,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-09 23:08:52,221] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default5]:[2022-09-09 23:08:52,215] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-09 23:08:52,215] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default3]:[2022-09-09 23:08:52,254] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-09 23:08:52,254] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default6]:[2022-09-09 23:08:52,196] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-09 23:08:52,196] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default2]:[2022-09-09 23:08:52,282] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-09 23:08:52,282] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default7]:[2022-09-09 23:08:52,359] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-09 23:08:52,359] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default6]:[2022-09-09 23:08:52,345] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-09 23:08:52,345] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default7]:[2022-09-09 23:08:52,290] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-09 23:08:52,291] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default5]:[2022-09-09 23:08:52,339] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-09 23:08:52,339] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default0]:[2022-09-09 23:08:52,299] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-09 23:08:52,299] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default7]:[2022-09-09 23:08:52,403] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-09 23:08:52,403] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default0]:[2022-09-09 23:08:52,400] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-09 23:08:52,401] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default5]:[2022-09-09 23:08:52,400] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-09 23:08:52,401] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default6]:[2022-09-09 23:08:52,401] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-09 23:08:52,401] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default5]:[2022-09-09 23:08:52,361] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-09 23:08:52,361] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default0]:[2022-09-09 23:08:52,446] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-09 23:08:52,446] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default7]:[2022-09-09 23:08:52,427] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-09 23:08:52,427] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default2]:[2022-09-09 23:08:52,465] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-09 23:08:52,465] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default3]:[2022-09-09 23:08:52,432] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-09 23:08:52,432] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default1]:[2022-09-09 23:08:52,469] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-09 23:08:52,469] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default0]:[2022-09-09 23:08:52,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-09 23:08:52,417] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default1]:[2022-09-09 23:08:52,497] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-09 23:08:52,497] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default1]:[2022-09-09 23:08:52,489] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-09 23:08:52,489] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default5]:[2022-09-09 23:08:52,454] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-09 23:08:52,454] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default6]:[2022-09-09 23:08:52,440] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-09 23:08:52,441] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default2]:[2022-09-09 23:08:52,512] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-09 23:08:52,512] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default3]:[2022-09-09 23:08:52,459] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-09 23:08:52,459] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default2]:[2022-09-09 23:08:52,537] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-09 23:08:52,537] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default3]:[2022-09-09 23:08:52,534] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-09 23:08:52,534] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default4]:[2022-09-09 23:08:52,537] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-09 23:08:52,537] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default0]:[2022-09-09 23:08:52,596] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-09 23:08:52,596] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default7]:[2022-09-09 23:08:52,594] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-09 23:08:52,594] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default4]:[2022-09-09 23:08:52,544] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-09 23:08:52,544] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default4]:[2022-09-09 23:08:52,549] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-09 23:08:52,549] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default6]:[2022-09-09 23:08:52,593] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-09 23:08:52,593] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default7]:[2022-09-09 23:08:52,564] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-09 23:08:52,564] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default6]:[2022-09-09 23:08:52,616] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-09 23:08:52,616] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default5]:[2022-09-09 23:08:52,603] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-09 23:08:52,604] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default4]:[2022-09-09 23:08:52,603] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-09 23:08:52,603] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default5]:[2022-09-09 23:08:52,665] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-09 23:08:52,665] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default2]:[2022-09-09 23:08:52,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-09 23:08:52,685] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default5]:[2022-09-09 23:08:52,683] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-09 23:08:52,683] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default1]:[2022-09-09 23:08:52,661] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-09 23:08:52,661] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default3]:[2022-09-09 23:08:52,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-09 23:08:52,636] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default2]:[2022-09-09 23:08:52,655] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-09 23:08:52,655] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default3]:[2022-09-09 23:08:52,665] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-09 23:08:52,665] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default0]:[2022-09-09 23:08:52,699] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-09 23:08:52,699] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default7]:[2022-09-09 23:08:52,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-09 23:08:52,685] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default7]:[2022-09-09 23:08:52,750] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-09 23:08:52,750] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default5]:[2022-09-09 23:08:52,746] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-09 23:08:52,746] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default5]:[2022-09-09 23:08:52,711] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-09 23:08:52,711] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default1]:[2022-09-09 23:08:52,714] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-09 23:08:52,714] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default1]:[2022-09-09 23:08:52,733] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-09 23:08:52,733] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default2]:[2022-09-09 23:08:52,806] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-09 23:08:52,806] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default4]:[2022-09-09 23:08:52,741] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-09 23:08:52,741] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default4]:[2022-09-09 23:08:52,767] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-09 23:08:52,767] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default5]:[2022-09-09 23:08:52,816] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-09 23:08:52,816] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default3]:[2022-09-09 23:08:52,771] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-09 23:08:52,771] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default5]:[2022-09-09 23:08:52,843] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-09 23:08:52,843] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default1]:[2022-09-09 23:08:52,798] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-09 23:08:52,798] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default3]:[2022-09-09 23:08:52,789] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-09 23:08:52,790] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default1]:[2022-09-09 23:08:52,872] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-09 23:08:52,872] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default1]:[2022-09-09 23:08:52,822] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-09 23:08:52,822] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default0]:[2022-09-09 23:08:52,824] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-09 23:08:52,824] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default4]:[2022-09-09 23:08:52,827] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-09 23:08:52,827] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default3]:[2022-09-09 23:08:52,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-09 23:08:52,887] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default0]:[2022-09-09 23:08:52,847] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-09 23:08:52,848] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default1]:[2022-09-09 23:08:52,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-09 23:08:52,932] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default6]:[2022-09-09 23:08:52,942] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-09 23:08:52,942] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default2]:[2022-09-09 23:08:52,941] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-09 23:08:52,941] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default5]:[2022-09-09 23:08:52,914] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-09 23:08:52,914] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default7]:[2022-09-09 23:08:52,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-09 23:08:52,910] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default7]:[2022-09-09 23:08:52,979] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-09 23:08:52,979] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default3]:[2022-09-09 23:08:53,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-09 23:08:53,023] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default4]:[2022-09-09 23:08:53,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-09 23:08:53,028] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default3]:[2022-09-09 23:08:53,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-09 23:08:53,020] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default6]:[2022-09-09 23:08:53,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-09 23:08:53,046] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default3]:[2022-09-09 23:08:53,048] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-09 23:08:53,048] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default5]:[2022-09-09 23:08:53,093] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-09 23:08:53,093] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default4]:[2022-09-09 23:08:53,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-09 23:08:53,097] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default4]:[2022-09-09 23:08:53,152] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-09 23:08:53,152] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default6]:[2022-09-09 23:08:53,151] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-09 23:08:53,152] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default4]:[2022-09-09 23:08:53,174] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-09 23:08:53,174] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default5]:[2022-09-09 23:08:53,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-09 23:08:53,221] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default0]:[2022-09-09 23:08:53,231] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-09 23:08:53,231] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default5]:[2022-09-09 23:08:53,204] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-09 23:08:53,204] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default2]:[2022-09-09 23:08:53,201] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-09 23:08:53,201] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default6]:[2022-09-09 23:08:53,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-09 23:08:53,186] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default6]:[2022-09-09 23:08:53,184] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-09 23:08:53,185] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default1]:[2022-09-09 23:08:53,249] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-09 23:08:53,249] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default6]:[2022-09-09 23:08:53,299] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-09 23:08:53,299] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default3]:[2022-09-09 23:08:53,285] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-09 23:08:53,286] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default1]:[2022-09-09 23:08:53,368] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-09 23:08:53,368] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default4]:[2022-09-09 23:08:53,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-09 23:08:53,407] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default2]:[2022-09-09 23:08:53,466] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-09 23:08:53,466] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default7]:[2022-09-09 23:08:53,490] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-09 23:08:53,490] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default7]:[2022-09-09 23:08:53,503] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-09 23:08:53,503] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default5]:[2022-09-09 23:08:53,505] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-09 23:08:53,505] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default4]:[2022-09-09 23:08:53,495] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-09 23:08:53,495] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default0]:[2022-09-09 23:08:53,515] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-09 23:08:53,515] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default2]:[2022-09-09 23:08:53,523] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-09 23:08:53,523] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default6]:[2022-09-09 23:08:53,569] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-09 23:08:53,569] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default4]:[2022-09-09 23:08:53,603] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-09 23:08:53,603] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default6]:[2022-09-09 23:08:53,570] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-09 23:08:53,570] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default5]:[2022-09-09 23:08:53,575] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-09 23:08:53,575] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default0]:[2022-09-09 23:08:53,614] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-09 23:08:53,614] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default4]:[2022-09-09 23:08:53,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-09 23:08:53,670] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default3]:[2022-09-09 23:08:53,704] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-09 23:08:53,705] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default1]:[2022-09-09 23:08:53,721] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-09 23:08:53,721] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default5]:[2022-09-09 23:08:53,658] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-09 23:08:53,658] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default6]:[2022-09-09 23:08:53,699] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-09 23:08:53,699] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default5]:[2022-09-09 23:08:53,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-09 23:08:53,757] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default3]:[2022-09-09 23:08:53,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-09 23:08:53,800] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default2]:[2022-09-09 23:08:53,846] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-09 23:08:53,846] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default0]:[2022-09-09 23:08:53,815] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-09 23:08:53,816] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default0]:[2022-09-09 23:08:53,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-09 23:08:53,783] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default0]:[2022-09-09 23:08:53,822] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-09 23:08:53,822] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default7]:[2022-09-09 23:08:53,847] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-09 23:08:53,847] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default3]:[2022-09-09 23:08:53,939] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-09 23:08:53,939] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default1]:[2022-09-09 23:08:53,977] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-09 23:08:53,977] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default7]:[2022-09-09 23:08:54,006] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-09 23:08:54,006] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default6]:[2022-09-09 23:08:54,085] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-09 23:08:54,085] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default6]:[2022-09-09 23:08:54,013] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-09 23:08:54,014] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default7]:[2022-09-09 23:08:54,139] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-09 23:08:54,139] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default4]:[2022-09-09 23:08:54,115] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-09 23:08:54,115] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default7]:[2022-09-09 23:08:54,155] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-09 23:08:54,156] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default6]:[2022-09-09 23:08:54,095] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-09 23:08:54,095] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default2]:[2022-09-09 23:08:54,220] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-09 23:08:54,220] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default7]:[2022-09-09 23:08:54,170] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-09 23:08:54,170] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default4]:[2022-09-09 23:08:54,201] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-09 23:08:54,201] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default7]:[2022-09-09 23:08:54,348] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-09 23:08:54,348] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default2]:[2022-09-09 23:08:54,449] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-09 23:08:54,450] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default1]:[2022-09-09 23:08:54,484] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-09 23:08:54,484] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default2]:[2022-09-09 23:08:54,577] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-09 23:08:54,578] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default0]:[2022-09-09 23:08:54,525] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-09 23:08:54,525] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default3]:[2022-09-09 23:08:54,504] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-09 23:08:54,504] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default5]:[2022-09-09 23:08:54,633] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-09 23:08:54,634] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default7]:[2022-09-09 23:08:54,666] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-09 23:08:54,666] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default4]:[2022-09-09 23:08:54,648] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-09 23:08:54,648] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default4]:[2022-09-09 23:08:54,719] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-09 23:08:54,719] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default5]:[2022-09-09 23:08:54,987] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-09 23:08:54,987] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default7]:[2022-09-09 23:08:55,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-09 23:08:55,024] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default1]:[2022-09-09 23:08:55,216] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-09 23:08:55,216] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default6]:[2022-09-09 23:08:55,238] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-09 23:08:55,238] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default3]:[2022-09-09 23:08:55,266] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-09 23:08:55,266] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default2]:[2022-09-09 23:08:55,299] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-09 23:08:55,299] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default0]:[2022-09-09 23:08:55,297] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-09 23:08:55,298] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default7]:[2022-09-09 23:08:55,373] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-09 23:08:55,374] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default6]:[2022-09-09 23:08:55,458] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-09 23:08:55,458] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default5]:[2022-09-09 23:08:55,500] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-09 23:08:55,500] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default6]:[2022-09-09 23:08:55,916] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-09 23:08:55,916] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default1]:[2022-09-09 23:08:56,000] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-09 23:08:56,000] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default4]:[2022-09-09 23:08:56,287] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-09 23:08:56,287] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default0]:[2022-09-09 23:08:56,250] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-09 23:08:56,250] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default7]:[2022-09-09 23:08:56,789] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-09 23:08:56,790] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default4]:[2022-09-09 23:08:56,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-09 23:08:56,808] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default3]:[2022-09-09 23:08:56,949] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-09 23:08:56,950] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default2]:[2022-09-09 23:08:57,007] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-09 23:08:57,007] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default2]:[2022-09-09 23:08:57,298] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-09 23:08:57,298] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default1]:[2022-09-09 23:08:57,649] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-09 23:08:57,649] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default0]:[2022-09-09 23:08:57,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-09 23:08:57,616] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default3]:[2022-09-09 23:08:57,708] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-09 23:08:57,708] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default0]:[2022-09-09 23:08:58,125] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-09 23:08:58,128] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default1]:[2022-09-09 23:08:59,188] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-09 23:08:59,188] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default1]:[2022-09-09 23:08:59,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-09 23:08:59,676] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default5]:[2022-09-09 23:08:59,961] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-09 23:08:59,961] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default4]:[2022-09-09 23:09:00,171] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-09 23:09:00,171] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default2]:[2022-09-09 23:09:00,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-09 23:09:00,305] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default7]:[2022-09-09 23:09:00,356] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-09 23:09:00,356] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default3]:[2022-09-09 23:09:00,361] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-09 23:09:00,361] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default1]:[2022-09-09 23:09:00,663] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-09 23:09:00,663] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default3]:[2022-09-09 23:09:00,693] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-09 23:09:00,693] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default0]:[2022-09-09 23:09:00,786] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-09 23:09:00,786] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default6]:[2022-09-09 23:09:00,954] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-09 23:09:00,954] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default0]:[2022-09-09 23:09:01,050] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-09 23:09:01,051] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default2]:[2022-09-09 23:09:01,307] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-09 23:09:01,307] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default1]:[2022-09-09 23:09:01,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-09 23:09:01,455] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default2]:[2022-09-09 23:09:01,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-09 23:09:01,980] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default3]:[2022-09-09 23:09:02,054] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-09 23:09:02,054] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default0]:[2022-09-09 23:09:02,096] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-09 23:09:02,096] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default1]:[2022-09-09 23:09:03,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-09 23:09:03,047] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default3]:[2022-09-09 23:09:03,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-09 23:09:03,156] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default0]:[2022-09-09 23:09:03,320] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-09 23:09:03,320] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default2]:[2022-09-09 23:09:03,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-09 23:09:03,313] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default4]:[2022-09-09 23:09:03,464] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-09 23:09:03,464] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default5]:[2022-09-09 23:09:04,035] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-09 23:09:04,035] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default7]:[2022-09-09 23:09:04,614] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-09 23:09:04,614] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default1]:[2022-09-09 23:09:04,844] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-09 23:09:04,844] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default0]:[2022-09-09 23:09:04,909] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-09 23:09:04,909] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default2]:[2022-09-09 23:09:04,989] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-09 23:09:04,989] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default3]:[2022-09-09 23:09:05,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-09 23:09:05,017] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default6]:[2022-09-09 23:09:04,958] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-09 23:09:04,958] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default4]:[2022-09-09 23:09:05,946] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-09 23:09:05,946] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default5]:[2022-09-09 23:09:06,331] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-09 23:09:06,331] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default7]:[2022-09-09 23:09:12,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-09 23:09:12,407] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default6]:[2022-09-09 23:09:12,568] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-09 23:09:12,568] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default7]:[2022-09-09 23:09:12,623] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-09 23:09:12,623] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default6]:[2022-09-09 23:09:12,575] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-09 23:09:12,575] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default4]:[2022-09-09 23:09:13,375] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-09 23:09:13,375] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default5]:[2022-09-09 23:09:13,381] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-09 23:09:13,382] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default7]:[2022-09-09 23:09:15,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-09 23:09:15,776] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default6]:[2022-09-09 23:09:15,912] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-09 23:09:15,912] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default5]:[2022-09-09 23:09:19,296] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-09 23:09:19,296] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]: successfully saved checkpoint at iteration 1245 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,411] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-09 23:09:19,412] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1245/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:time (ms) | save-checkpoint: 44573.17 -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default5]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default2]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default1]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default4]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default0]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default3]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default6]:[2022-09-09 23:09:19,412] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1245 is ready now! -[default7]: iteration 1246/ 3100 | consumed samples: 2551808 | consumed tokens: 5226102784 | elapsed time per iteration (s): 186.01 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.292927E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.010 | TFLOPs: 112.40 | -[default7]: iteration 1247/ 3100 | consumed samples: 2553856 | consumed tokens: 5230297088 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.317049E-01 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 1248/ 3100 | consumed samples: 2555904 | consumed tokens: 5234491392 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.364509E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.51 | -[default7]: iteration 1249/ 3100 | consumed samples: 2557952 | consumed tokens: 5238685696 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.369928E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 1250/ 3100 | consumed samples: 2560000 | consumed tokens: 5242880000 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.405512E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]:validation_pretraining loss at iteration 1250 | lm loss value: 2.401405E+00 | lm loss PPL: 1.103868E+01 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]: iteration 1251/ 3100 | consumed samples: 2562048 | consumed tokens: 5247074304 | elapsed time per iteration (s): 183.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.436229E-01 | grad norm: 0.436 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.169 | TFLOPs: 114.02 | -[default7]: iteration 1252/ 3100 | consumed samples: 2564096 | consumed tokens: 5251268608 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.366748E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 1253/ 3100 | consumed samples: 2566144 | consumed tokens: 5255462912 | elapsed time per iteration (s): 140.01 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.430972E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.627 | TFLOPs: 149.32 | -[default7]: iteration 1254/ 3100 | consumed samples: 2568192 | consumed tokens: 5259657216 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.324405E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 1255/ 3100 | consumed samples: 2570240 | consumed tokens: 5263851520 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.311595E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.582 | TFLOPs: 148.86 | -[default7]: iteration 1256/ 3100 | consumed samples: 2572288 | consumed tokens: 5268045824 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.284252E-01 | grad norm: 0.552 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1257/ 3100 | consumed samples: 2574336 | consumed tokens: 5272240128 | elapsed time per iteration (s): 140.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.283086E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.587 | TFLOPs: 148.91 | -[default7]: iteration 1258/ 3100 | consumed samples: 2576384 | consumed tokens: 5276434432 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.438630E-01 | grad norm: 0.382 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 1259/ 3100 | consumed samples: 2578432 | consumed tokens: 5280628736 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.249363E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 1260/ 3100 | consumed samples: 2580480 | consumed tokens: 5284823040 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.554367E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 1261/ 3100 | consumed samples: 2582528 | consumed tokens: 5289017344 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.417352E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 1262/ 3100 | consumed samples: 2584576 | consumed tokens: 5293211648 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.362751E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1263/ 3100 | consumed samples: 2586624 | consumed tokens: 5297405952 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.352693E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 1264/ 3100 | consumed samples: 2588672 | consumed tokens: 5301600256 | elapsed time per iteration (s): 140.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.391552E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.551 | TFLOPs: 148.54 | -[default7]: iteration 1265/ 3100 | consumed samples: 2590720 | consumed tokens: 5305794560 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.331171E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 1266/ 3100 | consumed samples: 2592768 | consumed tokens: 5309988864 | elapsed time per iteration (s): 140.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.456093E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.528 | TFLOPs: 148.31 | -[default7]: iteration 1267/ 3100 | consumed samples: 2594816 | consumed tokens: 5314183168 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.274723E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1268/ 3100 | consumed samples: 2596864 | consumed tokens: 5318377472 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.386928E-01 | grad norm: 0.516 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 1269/ 3100 | consumed samples: 2598912 | consumed tokens: 5322571776 | elapsed time per iteration (s): 141.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.371624E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.491 | TFLOPs: 147.93 | -[default7]: iteration 1270/ 3100 | consumed samples: 2600960 | consumed tokens: 5326766080 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.355397E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1271/ 3100 | consumed samples: 2603008 | consumed tokens: 5330960384 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.362612E-01 | grad norm: 0.426 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 1272/ 3100 | consumed samples: 2605056 | consumed tokens: 5335154688 | elapsed time per iteration (s): 141.11 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.443893E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.513 | TFLOPs: 148.16 | -[default7]: iteration 1273/ 3100 | consumed samples: 2607104 | consumed tokens: 5339348992 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.299487E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1274/ 3100 | consumed samples: 2609152 | consumed tokens: 5343543296 | elapsed time per iteration (s): 140.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.420832E-01 | grad norm: 0.449 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.570 | TFLOPs: 148.74 | -[default7]: iteration 1275/ 3100 | consumed samples: 2611200 | consumed tokens: 5347737600 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.350096E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1276/ 3100 | consumed samples: 2613248 | consumed tokens: 5351931904 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.318796E-01 | grad norm: 0.452 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 1277/ 3100 | consumed samples: 2615296 | consumed tokens: 5356126208 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.399185E-01 | grad norm: 0.438 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 1278/ 3100 | consumed samples: 2617344 | consumed tokens: 5360320512 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.288529E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 1279/ 3100 | consumed samples: 2619392 | consumed tokens: 5364514816 | elapsed time per iteration (s): 141.29 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.196362E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.495 | TFLOPs: 147.97 | -[default7]: iteration 1280/ 3100 | consumed samples: 2621440 | consumed tokens: 5368709120 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.278108E-01 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 1281/ 3100 | consumed samples: 2623488 | consumed tokens: 5372903424 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.327653E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.37 | -[default7]: iteration 1282/ 3100 | consumed samples: 2625536 | consumed tokens: 5377097728 | elapsed time per iteration (s): 140.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.381181E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.570 | TFLOPs: 148.74 | -[default7]: iteration 1283/ 3100 | consumed samples: 2627584 | consumed tokens: 5381292032 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.296262E-01 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.83 | -[default7]: iteration 1284/ 3100 | consumed samples: 2629632 | consumed tokens: 5385486336 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.505781E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.84 | -[default7]: iteration 1285/ 3100 | consumed samples: 2631680 | consumed tokens: 5389680640 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.343417E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.79 | -[default7]: iteration 1286/ 3100 | consumed samples: 2633728 | consumed tokens: 5393874944 | elapsed time per iteration (s): 140.99 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.427297E-01 | grad norm: 0.514 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.526 | TFLOPs: 148.28 | -[default7]: iteration 1287/ 3100 | consumed samples: 2635776 | consumed tokens: 5398069248 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.246954E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.31 | -[default7]: iteration 1288/ 3100 | consumed samples: 2637824 | consumed tokens: 5402263552 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.382087E-01 | grad norm: 1.071 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1289/ 3100 | consumed samples: 2639872 | consumed tokens: 5406457856 | elapsed time per iteration (s): 141.23 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.344222E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.501 | TFLOPs: 148.03 | -[default7]: iteration 1290/ 3100 | consumed samples: 2641920 | consumed tokens: 5410652160 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.325263E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 1291/ 3100 | consumed samples: 2643968 | consumed tokens: 5414846464 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.431084E-01 | grad norm: 0.602 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 1292/ 3100 | consumed samples: 2646016 | consumed tokens: 5419040768 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.336381E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 1293/ 3100 | consumed samples: 2648064 | consumed tokens: 5423235072 | elapsed time per iteration (s): 141.15 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.345707E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.510 | TFLOPs: 148.12 | -[default7]: iteration 1294/ 3100 | consumed samples: 2650112 | consumed tokens: 5427429376 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.302144E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1295/ 3100 | consumed samples: 2652160 | consumed tokens: 5431623680 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.251725E-01 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1296/ 3100 | consumed samples: 2654208 | consumed tokens: 5435817984 | elapsed time per iteration (s): 140.08 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.368838E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.621 | TFLOPs: 149.25 | -[default7]: iteration 1297/ 3100 | consumed samples: 2656256 | consumed tokens: 5440012288 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.319684E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 1298/ 3100 | consumed samples: 2658304 | consumed tokens: 5444206592 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.312764E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.82 | -[default7]: iteration 1299/ 3100 | consumed samples: 2660352 | consumed tokens: 5448400896 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.376981E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1300/ 3100 | consumed samples: 2662400 | consumed tokens: 5452595200 | elapsed time per iteration (s): 140.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.329813E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.572 | TFLOPs: 148.75 | -[default7]: iteration 1301/ 3100 | consumed samples: 2664448 | consumed tokens: 5456789504 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.221405E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 1302/ 3100 | consumed samples: 2666496 | consumed tokens: 5460983808 | elapsed time per iteration (s): 141.16 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.202884E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.509 | TFLOPs: 148.11 | -[default7]: iteration 1303/ 3100 | consumed samples: 2668544 | consumed tokens: 5465178112 | elapsed time per iteration (s): 142.05 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.249316E-01 | grad norm: 0.365 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.418 | TFLOPs: 147.18 | -[default7]: iteration 1304/ 3100 | consumed samples: 2670592 | consumed tokens: 5469372416 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.196872E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1305/ 3100 | consumed samples: 2672640 | consumed tokens: 5473566720 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.349501E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 1306/ 3100 | consumed samples: 2674688 | consumed tokens: 5477761024 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.309445E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1307/ 3100 | consumed samples: 2676736 | consumed tokens: 5481955328 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.340923E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1308/ 3100 | consumed samples: 2678784 | consumed tokens: 5486149632 | elapsed time per iteration (s): 140.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.186656E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.571 | TFLOPs: 148.75 | -[default7]: iteration 1309/ 3100 | consumed samples: 2680832 | consumed tokens: 5490343936 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.313812E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.48 | -[default7]: iteration 1310/ 3100 | consumed samples: 2682880 | consumed tokens: 5494538240 | elapsed time per iteration (s): 141.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.210368E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.82 | -[default7]: iteration 1311/ 3100 | consumed samples: 2684928 | consumed tokens: 5498732544 | elapsed time per iteration (s): 141.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.225193E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.484 | TFLOPs: 147.86 | -[default7]: iteration 1312/ 3100 | consumed samples: 2686976 | consumed tokens: 5502926848 | elapsed time per iteration (s): 141.21 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.346651E-01 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.503 | TFLOPs: 148.06 | -[default7]: iteration 1313/ 3100 | consumed samples: 2689024 | consumed tokens: 5507121152 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.186347E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.49 | -[default7]: iteration 1314/ 3100 | consumed samples: 2691072 | consumed tokens: 5511315456 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.268129E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1315/ 3100 | consumed samples: 2693120 | consumed tokens: 5515509760 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.055982E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 1316/ 3100 | consumed samples: 2695168 | consumed tokens: 5519704064 | elapsed time per iteration (s): 140.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.207678E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.538 | TFLOPs: 148.41 | -[default7]: iteration 1317/ 3100 | consumed samples: 2697216 | consumed tokens: 5523898368 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.247153E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1318/ 3100 | consumed samples: 2699264 | consumed tokens: 5528092672 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.344034E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 1319/ 3100 | consumed samples: 2701312 | consumed tokens: 5532286976 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.429646E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.27 | -[default7]: iteration 1320/ 3100 | consumed samples: 2703360 | consumed tokens: 5536481280 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.186915E-01 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1321/ 3100 | consumed samples: 2705408 | consumed tokens: 5540675584 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.364643E-01 | grad norm: 0.523 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.45 | -[default7]: iteration 1322/ 3100 | consumed samples: 2707456 | consumed tokens: 5544869888 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.396544E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1323/ 3100 | consumed samples: 2709504 | consumed tokens: 5549064192 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.142309E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 1324/ 3100 | consumed samples: 2711552 | consumed tokens: 5553258496 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.395064E-01 | grad norm: 0.788 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1325/ 3100 | consumed samples: 2713600 | consumed tokens: 5557452800 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.326389E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1326/ 3100 | consumed samples: 2715648 | consumed tokens: 5561647104 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.355228E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1327/ 3100 | consumed samples: 2717696 | consumed tokens: 5565841408 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.333943E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1328/ 3100 | consumed samples: 2719744 | consumed tokens: 5570035712 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.257241E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 1329/ 3100 | consumed samples: 2721792 | consumed tokens: 5574230016 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.188565E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1330/ 3100 | consumed samples: 2723840 | consumed tokens: 5578424320 | elapsed time per iteration (s): 141.10 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.210422E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.514 | TFLOPs: 148.17 | -[default7]: iteration 1331/ 3100 | consumed samples: 2725888 | consumed tokens: 5582618624 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.364667E-01 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1332/ 3100 | consumed samples: 2727936 | consumed tokens: 5586812928 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.340735E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1333/ 3100 | consumed samples: 2729984 | consumed tokens: 5591007232 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.260266E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1334/ 3100 | consumed samples: 2732032 | consumed tokens: 5595201536 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.303097E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 1335/ 3100 | consumed samples: 2734080 | consumed tokens: 5599395840 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.229212E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1336/ 3100 | consumed samples: 2736128 | consumed tokens: 5603590144 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.387970E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1337/ 3100 | consumed samples: 2738176 | consumed tokens: 5607784448 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.335652E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 1338/ 3100 | consumed samples: 2740224 | consumed tokens: 5611978752 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.343092E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1339/ 3100 | consumed samples: 2742272 | consumed tokens: 5616173056 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.279966E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1340/ 3100 | consumed samples: 2744320 | consumed tokens: 5620367360 | elapsed time per iteration (s): 140.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.163764E-01 | grad norm: 0.478 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.576 | TFLOPs: 148.80 | -[default7]: iteration 1341/ 3100 | consumed samples: 2746368 | consumed tokens: 5624561664 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.306744E-01 | grad norm: 0.469 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 1342/ 3100 | consumed samples: 2748416 | consumed tokens: 5628755968 | elapsed time per iteration (s): 141.16 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.200058E-01 | grad norm: 1.206 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.509 | TFLOPs: 148.11 | -[default7]: iteration 1343/ 3100 | consumed samples: 2750464 | consumed tokens: 5632950272 | elapsed time per iteration (s): 141.18 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.164775E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.506 | TFLOPs: 148.09 | -[default7]: iteration 1344/ 3100 | consumed samples: 2752512 | consumed tokens: 5637144576 | elapsed time per iteration (s): 141.16 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.324419E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.508 | TFLOPs: 148.10 | -[default7]: iteration 1345/ 3100 | consumed samples: 2754560 | consumed tokens: 5641338880 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.306953E-01 | grad norm: 0.365 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 1346/ 3100 | consumed samples: 2756608 | consumed tokens: 5645533184 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.389989E-01 | grad norm: 0.413 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1347/ 3100 | consumed samples: 2758656 | consumed tokens: 5649727488 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.307580E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1348/ 3100 | consumed samples: 2760704 | consumed tokens: 5653921792 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.305521E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 1349/ 3100 | consumed samples: 2762752 | consumed tokens: 5658116096 | elapsed time per iteration (s): 140.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.407497E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.577 | TFLOPs: 148.80 | -[default7]: iteration 1350/ 3100 | consumed samples: 2764800 | consumed tokens: 5662310400 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.243483E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 1351/ 3100 | consumed samples: 2766848 | consumed tokens: 5666504704 | elapsed time per iteration (s): 141.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.200284E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.429 | TFLOPs: 147.29 | -[default7]: iteration 1352/ 3100 | consumed samples: 2768896 | consumed tokens: 5670699008 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.344017E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 1353/ 3100 | consumed samples: 2770944 | consumed tokens: 5674893312 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.256350E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1354/ 3100 | consumed samples: 2772992 | consumed tokens: 5679087616 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.005609E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.34 | -[default7]: iteration 1355/ 3100 | consumed samples: 2775040 | consumed tokens: 5683281920 | elapsed time per iteration (s): 141.12 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.245825E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.513 | TFLOPs: 148.15 | -[default7]: iteration 1356/ 3100 | consumed samples: 2777088 | consumed tokens: 5687476224 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.299338E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 1357/ 3100 | consumed samples: 2779136 | consumed tokens: 5691670528 | elapsed time per iteration (s): 140.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.289118E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.550 | TFLOPs: 148.53 | -[default7]: iteration 1358/ 3100 | consumed samples: 2781184 | consumed tokens: 5695864832 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.200188E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1359/ 3100 | consumed samples: 2783232 | consumed tokens: 5700059136 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.188708E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1360/ 3100 | consumed samples: 2785280 | consumed tokens: 5704253440 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.335842E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 1361/ 3100 | consumed samples: 2787328 | consumed tokens: 5708447744 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.304589E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.45 | -[default7]: iteration 1362/ 3100 | consumed samples: 2789376 | consumed tokens: 5712642048 | elapsed time per iteration (s): 141.99 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.197937E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.424 | TFLOPs: 147.24 | -[default7]: iteration 1363/ 3100 | consumed samples: 2791424 | consumed tokens: 5716836352 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.161023E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1364/ 3100 | consumed samples: 2793472 | consumed tokens: 5721030656 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.176748E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1365/ 3100 | consumed samples: 2795520 | consumed tokens: 5725224960 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.159603E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1366/ 3100 | consumed samples: 2797568 | consumed tokens: 5729419264 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.175208E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 1367/ 3100 | consumed samples: 2799616 | consumed tokens: 5733613568 | elapsed time per iteration (s): 141.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.222005E-01 | grad norm: 0.875 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.429 | TFLOPs: 147.30 | -[default7]: iteration 1368/ 3100 | consumed samples: 2801664 | consumed tokens: 5737807872 | elapsed time per iteration (s): 141.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.275030E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.428 | TFLOPs: 147.28 | -[default7]: iteration 1369/ 3100 | consumed samples: 2803712 | consumed tokens: 5742002176 | elapsed time per iteration (s): 141.38 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.265527E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.486 | TFLOPs: 147.88 | -[default7]: iteration 1370/ 3100 | consumed samples: 2805760 | consumed tokens: 5746196480 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.158834E-01 | grad norm: 0.557 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 1371/ 3100 | consumed samples: 2807808 | consumed tokens: 5750390784 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.182660E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 1372/ 3100 | consumed samples: 2809856 | consumed tokens: 5754585088 | elapsed time per iteration (s): 141.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.158084E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.427 | TFLOPs: 147.28 | -[default7]: iteration 1373/ 3100 | consumed samples: 2811904 | consumed tokens: 5758779392 | elapsed time per iteration (s): 140.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.149064E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.575 | TFLOPs: 148.79 | -[default7]: iteration 1374/ 3100 | consumed samples: 2813952 | consumed tokens: 5762973696 | elapsed time per iteration (s): 140.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.281487E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.542 | TFLOPs: 148.45 | -[default7]: iteration 1375/ 3100 | consumed samples: 2816000 | consumed tokens: 5767168000 | elapsed time per iteration (s): 141.30 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.221010E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.494 | TFLOPs: 147.96 | -[default7]: iteration 1376/ 3100 | consumed samples: 2818048 | consumed tokens: 5771362304 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.226096E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1377/ 3100 | consumed samples: 2820096 | consumed tokens: 5775556608 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.324571E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.35 | -[default7]: iteration 1378/ 3100 | consumed samples: 2822144 | consumed tokens: 5779750912 | elapsed time per iteration (s): 141.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.092186E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.488 | TFLOPs: 147.90 | -[default7]: iteration 1379/ 3100 | consumed samples: 2824192 | consumed tokens: 5783945216 | elapsed time per iteration (s): 142.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.235467E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.420 | TFLOPs: 147.20 | -[default7]: iteration 1380/ 3100 | consumed samples: 2826240 | consumed tokens: 5788139520 | elapsed time per iteration (s): 142.01 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.159234E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.421 | TFLOPs: 147.22 | -[default7]: iteration 1381/ 3100 | consumed samples: 2828288 | consumed tokens: 5792333824 | elapsed time per iteration (s): 141.31 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.319243E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.493 | TFLOPs: 147.95 | -[default7]: iteration 1382/ 3100 | consumed samples: 2830336 | consumed tokens: 5796528128 | elapsed time per iteration (s): 141.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.159858E-01 | grad norm: 0.739 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.430 | TFLOPs: 147.31 | -[default7]: iteration 1383/ 3100 | consumed samples: 2832384 | consumed tokens: 5800722432 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.320699E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1384/ 3100 | consumed samples: 2834432 | consumed tokens: 5804916736 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.158681E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.425 | TFLOPs: 147.26 | -[default7]: iteration 1385/ 3100 | consumed samples: 2836480 | consumed tokens: 5809111040 | elapsed time per iteration (s): 140.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.273326E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.562 | TFLOPs: 148.66 | -[default7]: iteration 1386/ 3100 | consumed samples: 2838528 | consumed tokens: 5813305344 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.321936E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1387/ 3100 | consumed samples: 2840576 | consumed tokens: 5817499648 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.108340E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 1388/ 3100 | consumed samples: 2842624 | consumed tokens: 5821693952 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.151494E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 1389/ 3100 | consumed samples: 2844672 | consumed tokens: 5825888256 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.106420E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.27 | -[default7]: iteration 1390/ 3100 | consumed samples: 2846720 | consumed tokens: 5830082560 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.231629E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.582 | TFLOPs: 148.85 | -[default7]: iteration 1391/ 3100 | consumed samples: 2848768 | consumed tokens: 5834276864 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.064488E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 1392/ 3100 | consumed samples: 2850816 | consumed tokens: 5838471168 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.241544E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.425 | TFLOPs: 147.26 | -[default7]: iteration 1393/ 3100 | consumed samples: 2852864 | consumed tokens: 5842665472 | elapsed time per iteration (s): 140.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.142214E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.532 | TFLOPs: 148.35 | -[default7]: iteration 1394/ 3100 | consumed samples: 2854912 | consumed tokens: 5846859776 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.234841E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1395/ 3100 | consumed samples: 2856960 | consumed tokens: 5851054080 | elapsed time per iteration (s): 140.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.959579E-01 | grad norm: 0.382 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.569 | TFLOPs: 148.73 | -[default7]: iteration 1396/ 3100 | consumed samples: 2859008 | consumed tokens: 5855248384 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.143873E-01 | grad norm: 1.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1397/ 3100 | consumed samples: 2861056 | consumed tokens: 5859442688 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.123745E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 1398/ 3100 | consumed samples: 2863104 | consumed tokens: 5863636992 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.291785E-01 | grad norm: 0.445 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1399/ 3100 | consumed samples: 2865152 | consumed tokens: 5867831296 | elapsed time per iteration (s): 141.27 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.143495E-01 | grad norm: 0.557 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.497 | TFLOPs: 147.99 | -[default7]: iteration 1400/ 3100 | consumed samples: 2867200 | consumed tokens: 5872025600 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.075224E-01 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 1401/ 3100 | consumed samples: 2869248 | consumed tokens: 5876219904 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.257730E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 1402/ 3100 | consumed samples: 2871296 | consumed tokens: 5880414208 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.186656E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 1403/ 3100 | consumed samples: 2873344 | consumed tokens: 5884608512 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.091596E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1404/ 3100 | consumed samples: 2875392 | consumed tokens: 5888802816 | elapsed time per iteration (s): 140.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.162870E-01 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.540 | TFLOPs: 148.43 | -[default7]: iteration 1405/ 3100 | consumed samples: 2877440 | consumed tokens: 5892997120 | elapsed time per iteration (s): 141.34 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.130909E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.490 | TFLOPs: 147.92 | -[default7]: iteration 1406/ 3100 | consumed samples: 2879488 | consumed tokens: 5897191424 | elapsed time per iteration (s): 140.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.167470E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.547 | TFLOPs: 148.50 | -[default7]: iteration 1407/ 3100 | consumed samples: 2881536 | consumed tokens: 5901385728 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.241731E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1408/ 3100 | consumed samples: 2883584 | consumed tokens: 5905580032 | elapsed time per iteration (s): 141.34 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.103019E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.490 | TFLOPs: 147.92 | -[default7]: iteration 1409/ 3100 | consumed samples: 2885632 | consumed tokens: 5909774336 | elapsed time per iteration (s): 141.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.205390E-01 | grad norm: 0.413 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.522 | TFLOPs: 148.25 | -[default7]: iteration 1410/ 3100 | consumed samples: 2887680 | consumed tokens: 5913968640 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.321180E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1411/ 3100 | consumed samples: 2889728 | consumed tokens: 5918162944 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.166032E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1412/ 3100 | consumed samples: 2891776 | consumed tokens: 5922357248 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.086006E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1413/ 3100 | consumed samples: 2893824 | consumed tokens: 5926551552 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.072261E-01 | grad norm: 0.549 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1414/ 3100 | consumed samples: 2895872 | consumed tokens: 5930745856 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.193438E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.500 | TFLOPs: 148.02 | -[default7]: iteration 1415/ 3100 | consumed samples: 2897920 | consumed tokens: 5934940160 | elapsed time per iteration (s): 141.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.127404E-01 | grad norm: 0.539 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.488 | TFLOPs: 147.90 | -[default7]: iteration 1416/ 3100 | consumed samples: 2899968 | consumed tokens: 5939134464 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.160149E-01 | grad norm: 1.863 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 1417/ 3100 | consumed samples: 2902016 | consumed tokens: 5943328768 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.257494E-01 | grad norm: 0.484 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1418/ 3100 | consumed samples: 2904064 | consumed tokens: 5947523072 | elapsed time per iteration (s): 141.22 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.336867E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.502 | TFLOPs: 148.05 | -[default7]: iteration 1419/ 3100 | consumed samples: 2906112 | consumed tokens: 5951717376 | elapsed time per iteration (s): 140.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.277283E-01 | grad norm: 1.311 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.584 | TFLOPs: 148.88 | -[default7]: iteration 1420/ 3100 | consumed samples: 2908160 | consumed tokens: 5955911680 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.080025E-01 | grad norm: 0.370 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1421/ 3100 | consumed samples: 2910208 | consumed tokens: 5960105984 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.143572E-01 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.82 | -[default7]: iteration 1422/ 3100 | consumed samples: 2912256 | consumed tokens: 5964300288 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.002216E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 1423/ 3100 | consumed samples: 2914304 | consumed tokens: 5968494592 | elapsed time per iteration (s): 141.09 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.073972E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.515 | TFLOPs: 148.18 | -[default7]: iteration 1424/ 3100 | consumed samples: 2916352 | consumed tokens: 5972688896 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.246717E-01 | grad norm: 0.858 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.82 | -[default7]: iteration 1425/ 3100 | consumed samples: 2918400 | consumed tokens: 5976883200 | elapsed time per iteration (s): 140.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.077325E-01 | grad norm: 0.426 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.564 | TFLOPs: 148.67 | -[default7]: iteration 1426/ 3100 | consumed samples: 2920448 | consumed tokens: 5981077504 | elapsed time per iteration (s): 140.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.167601E-01 | grad norm: 0.656 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.564 | TFLOPs: 148.68 | -[default7]: iteration 1427/ 3100 | consumed samples: 2922496 | consumed tokens: 5985271808 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.304089E-01 | grad norm: 0.815 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 1428/ 3100 | consumed samples: 2924544 | consumed tokens: 5989466112 | elapsed time per iteration (s): 141.19 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.189490E-01 | grad norm: 0.501 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.506 | TFLOPs: 148.08 | -[default7]: iteration 1429/ 3100 | consumed samples: 2926592 | consumed tokens: 5993660416 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.124158E-01 | grad norm: 0.491 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1430/ 3100 | consumed samples: 2928640 | consumed tokens: 5997854720 | elapsed time per iteration (s): 142.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.971792E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.421 | TFLOPs: 147.21 | -[default7]: iteration 1431/ 3100 | consumed samples: 2930688 | consumed tokens: 6002049024 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.028932E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.73 | -[default7]: iteration 1432/ 3100 | consumed samples: 2932736 | consumed tokens: 6006243328 | elapsed time per iteration (s): 141.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.127778E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.430 | TFLOPs: 147.30 | -[default7]: iteration 1433/ 3100 | consumed samples: 2934784 | consumed tokens: 6010437632 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.054642E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 1434/ 3100 | consumed samples: 2936832 | consumed tokens: 6014631936 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.022264E-01 | grad norm: 0.370 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1435/ 3100 | consumed samples: 2938880 | consumed tokens: 6018826240 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.090513E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1436/ 3100 | consumed samples: 2940928 | consumed tokens: 6023020544 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.178259E-01 | grad norm: 4.832 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 1437/ 3100 | consumed samples: 2942976 | consumed tokens: 6027214848 | elapsed time per iteration (s): 140.30 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.297710E-01 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.597 | TFLOPs: 149.01 | -[default7]: iteration 1438/ 3100 | consumed samples: 2945024 | consumed tokens: 6031409152 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.990193E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 1439/ 3100 | consumed samples: 2947072 | consumed tokens: 6035603456 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.144475E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1440/ 3100 | consumed samples: 2949120 | consumed tokens: 6039797760 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.100479E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.499 | TFLOPs: 148.01 | -[default7]: iteration 1441/ 3100 | consumed samples: 2951168 | consumed tokens: 6043992064 | elapsed time per iteration (s): 141.09 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.162431E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.516 | TFLOPs: 148.18 | -[default7]: iteration 1442/ 3100 | consumed samples: 2953216 | consumed tokens: 6048186368 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.162264E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1443/ 3100 | consumed samples: 2955264 | consumed tokens: 6052380672 | elapsed time per iteration (s): 142.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.124490E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.420 | TFLOPs: 147.20 | -[default7]: iteration 1444/ 3100 | consumed samples: 2957312 | consumed tokens: 6056574976 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.154988E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1445/ 3100 | consumed samples: 2959360 | consumed tokens: 6060769280 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.113245E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.27 | -[default7]: iteration 1446/ 3100 | consumed samples: 2961408 | consumed tokens: 6064963584 | elapsed time per iteration (s): 141.35 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.019443E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.488 | TFLOPs: 147.90 | -[default7]: iteration 1447/ 3100 | consumed samples: 2963456 | consumed tokens: 6069157888 | elapsed time per iteration (s): 141.11 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.086190E-01 | grad norm: 0.787 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.514 | TFLOPs: 148.16 | -[default7]: iteration 1448/ 3100 | consumed samples: 2965504 | consumed tokens: 6073352192 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.026236E-01 | grad norm: 0.430 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 1449/ 3100 | consumed samples: 2967552 | consumed tokens: 6077546496 | elapsed time per iteration (s): 141.19 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.273605E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.506 | TFLOPs: 148.08 | -[default7]: iteration 1450/ 3100 | consumed samples: 2969600 | consumed tokens: 6081740800 | elapsed time per iteration (s): 141.99 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.937511E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.423 | TFLOPs: 147.24 | -[default7]: iteration 1451/ 3100 | consumed samples: 2971648 | consumed tokens: 6085935104 | elapsed time per iteration (s): 141.26 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.025354E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.498 | TFLOPs: 148.00 | -[default7]: iteration 1452/ 3100 | consumed samples: 2973696 | consumed tokens: 6090129408 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.161189E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 1453/ 3100 | consumed samples: 2975744 | consumed tokens: 6094323712 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.102569E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.425 | TFLOPs: 147.26 | -[default7]: iteration 1454/ 3100 | consumed samples: 2977792 | consumed tokens: 6098518016 | elapsed time per iteration (s): 142.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.112577E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.421 | TFLOPs: 147.21 | -[default7]: iteration 1455/ 3100 | consumed samples: 2979840 | consumed tokens: 6102712320 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.131853E-01 | grad norm: 0.431 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 1456/ 3100 | consumed samples: 2981888 | consumed tokens: 6106906624 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.077418E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 1457/ 3100 | consumed samples: 2983936 | consumed tokens: 6111100928 | elapsed time per iteration (s): 141.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.135028E-01 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.491 | TFLOPs: 147.93 | -[default7]: iteration 1458/ 3100 | consumed samples: 2985984 | consumed tokens: 6115295232 | elapsed time per iteration (s): 140.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.116037E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.542 | TFLOPs: 148.45 | -[default7]: iteration 1459/ 3100 | consumed samples: 2988032 | consumed tokens: 6119489536 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.066764E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.73 | -[default7]: iteration 1460/ 3100 | consumed samples: 2990080 | consumed tokens: 6123683840 | elapsed time per iteration (s): 141.20 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.017516E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.504 | TFLOPs: 148.06 | -[default7]: iteration 1461/ 3100 | consumed samples: 2992128 | consumed tokens: 6127878144 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.064330E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1462/ 3100 | consumed samples: 2994176 | consumed tokens: 6132072448 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.002880E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1463/ 3100 | consumed samples: 2996224 | consumed tokens: 6136266752 | elapsed time per iteration (s): 141.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.034968E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.428 | TFLOPs: 147.29 | -[default7]: iteration 1464/ 3100 | consumed samples: 2998272 | consumed tokens: 6140461056 | elapsed time per iteration (s): 141.32 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.038080E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.492 | TFLOPs: 147.94 | -[default7]: iteration 1465/ 3100 | consumed samples: 3000320 | consumed tokens: 6144655360 | elapsed time per iteration (s): 140.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.042183E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.568 | TFLOPs: 148.72 | -[default7]: iteration 1466/ 3100 | consumed samples: 3002368 | consumed tokens: 6148849664 | elapsed time per iteration (s): 141.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.965378E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.427 | TFLOPs: 147.28 | -[default7]: iteration 1467/ 3100 | consumed samples: 3004416 | consumed tokens: 6153043968 | elapsed time per iteration (s): 141.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.164496E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.488 | TFLOPs: 147.90 | -[default7]: iteration 1468/ 3100 | consumed samples: 3006464 | consumed tokens: 6157238272 | elapsed time per iteration (s): 140.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.960278E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.542 | TFLOPs: 148.45 | -[default7]: iteration 1469/ 3100 | consumed samples: 3008512 | consumed tokens: 6161432576 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.214483E-01 | grad norm: 0.382 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1470/ 3100 | consumed samples: 3010560 | consumed tokens: 6165626880 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.998109E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.45 | -[default7]: iteration 1471/ 3100 | consumed samples: 3012608 | consumed tokens: 6169821184 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.049783E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1472/ 3100 | consumed samples: 3014656 | consumed tokens: 6174015488 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.189332E-01 | grad norm: 0.471 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1473/ 3100 | consumed samples: 3016704 | consumed tokens: 6178209792 | elapsed time per iteration (s): 140.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.087399E-01 | grad norm: 0.365 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.575 | TFLOPs: 148.79 | -[default7]: iteration 1474/ 3100 | consumed samples: 3018752 | consumed tokens: 6182404096 | elapsed time per iteration (s): 141.26 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.029014E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.498 | TFLOPs: 148.00 | -[default7]: iteration 1475/ 3100 | consumed samples: 3020800 | consumed tokens: 6186598400 | elapsed time per iteration (s): 141.01 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.128616E-01 | grad norm: 0.430 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.524 | TFLOPs: 148.26 | -[default7]: iteration 1476/ 3100 | consumed samples: 3022848 | consumed tokens: 6190792704 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.979062E-01 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 1477/ 3100 | consumed samples: 3024896 | consumed tokens: 6194987008 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.971987E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1478/ 3100 | consumed samples: 3026944 | consumed tokens: 6199181312 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.869521E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1479/ 3100 | consumed samples: 3028992 | consumed tokens: 6203375616 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.019699E-01 | grad norm: 0.353 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 1480/ 3100 | consumed samples: 3031040 | consumed tokens: 6207569920 | elapsed time per iteration (s): 140.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.875705E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.557 | TFLOPs: 148.61 | -[default7]: iteration 1481/ 3100 | consumed samples: 3033088 | consumed tokens: 6211764224 | elapsed time per iteration (s): 140.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.163723E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.559 | TFLOPs: 148.62 | -[default7]: iteration 1482/ 3100 | consumed samples: 3035136 | consumed tokens: 6215958528 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.921420E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 1483/ 3100 | consumed samples: 3037184 | consumed tokens: 6220152832 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.972585E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.427 | TFLOPs: 147.28 | -[default7]: iteration 1484/ 3100 | consumed samples: 3039232 | consumed tokens: 6224347136 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.097536E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 1485/ 3100 | consumed samples: 3041280 | consumed tokens: 6228541440 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.061942E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 1486/ 3100 | consumed samples: 3043328 | consumed tokens: 6232735744 | elapsed time per iteration (s): 140.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.097597E-01 | grad norm: 0.454 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.564 | TFLOPs: 148.67 | -[default7]: iteration 1487/ 3100 | consumed samples: 3045376 | consumed tokens: 6236930048 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.154399E-01 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.47 | -[default7]: iteration 1488/ 3100 | consumed samples: 3047424 | consumed tokens: 6241124352 | elapsed time per iteration (s): 140.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.993743E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.574 | TFLOPs: 148.78 | -[default7]: iteration 1489/ 3100 | consumed samples: 3049472 | consumed tokens: 6245318656 | elapsed time per iteration (s): 142.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.972359E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.423 | TFLOPs: 147.24 | -[default7]: iteration 1490/ 3100 | consumed samples: 3051520 | consumed tokens: 6249512960 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.952572E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1491/ 3100 | consumed samples: 3053568 | consumed tokens: 6253707264 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.056496E-01 | grad norm: 0.382 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 1492/ 3100 | consumed samples: 3055616 | consumed tokens: 6257901568 | elapsed time per iteration (s): 141.15 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.033099E-01 | grad norm: 0.437 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.509 | TFLOPs: 148.12 | -[default7]: iteration 1493/ 3100 | consumed samples: 3057664 | consumed tokens: 6262095872 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.137177E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.499 | TFLOPs: 148.01 | -[default0]:saving checkpoint at iteration 1494 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-10 08:57:14,175] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1494 is begin to save! -[default7]: iteration 1494/ 3100 | consumed samples: 3059712 | consumed tokens: 6266290176 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.950060E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.430 | TFLOPs: 147.31 | -[default0]:[2022-09-10 08:57:14,213] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_30-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,274] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_29-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_08-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_33-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_26-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,201] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_35-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,213] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_31-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,200] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_34-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_48-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,275] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_61-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_22-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,275] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_49-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_44-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_12-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_36-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_37-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,279] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_01-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_58-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_59-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,275] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_54-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,286] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_67-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,274] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_52-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_46-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_63-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_42-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,277] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_23-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,274] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_53-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_62-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,286] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_04-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_07-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_43-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_47-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_24-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_03-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_51-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_45-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_38-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_20-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_32-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_41-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_11-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_19-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_18-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_40-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,274] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_28-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,286] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_57-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_50-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,275] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_55-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,275] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_60-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_06-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,286] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_56-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_66-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,286] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_15-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,286] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_14-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_13-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,275] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_17-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,324] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_69-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,345] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_72-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_10-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,275] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_16-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,324] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_68-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,338] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_71_model_states.pt... -[default4]:[2022-09-10 08:57:14,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_71_model_states.pt. -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_27-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_64-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_65-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,324] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_71-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_39-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_25-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_21-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,276] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_09-model_00-model_states.pt... -[default4]:[2022-09-10 08:57:14,286] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_05-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:14,314] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_70-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:17,476] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_30-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,477] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_28_model_states.pt... -[default0]:[2022-09-10 08:57:17,487] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_28_model_states.pt. -[default0]:[2022-09-10 08:57:17,599] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_40-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,599] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_38_model_states.pt... -[default0]:[2022-09-10 08:57:17,607] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_38_model_states.pt. -[default0]:[2022-09-10 08:57:17,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_66-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,636] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_64_model_states.pt... -[default0]:[2022-09-10 08:57:17,639] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_64_model_states.pt. -[default4]:[2022-09-10 08:57:17,621] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_31-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:17,621] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_29_model_states.pt... -[default4]:[2022-09-10 08:57:17,623] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_29_model_states.pt. -[default0]:[2022-09-10 08:57:17,645] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_32-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,645] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_30_model_states.pt... -[default0]:[2022-09-10 08:57:17,665] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_30_model_states.pt. -[default0]:[2022-09-10 08:57:17,706] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_72-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,707] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_74-model_00-model_states.pt... -[default0]:[2022-09-10 08:57:17,713] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_74-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,713] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_70_model_states.pt... -[default0]:[2022-09-10 08:57:17,715] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_70_model_states.pt. -[default0]:[2022-09-10 08:57:17,690] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_16-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,690] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_14_model_states.pt... -[default0]:[2022-09-10 08:57:17,698] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_14_model_states.pt. -[default4]:[2022-09-10 08:57:17,807] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_65-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:17,808] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_63_model_states.pt... -[default4]:[2022-09-10 08:57:17,816] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_63_model_states.pt. -[default4]:[2022-09-10 08:57:17,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_21-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:17,857] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_19_model_states.pt... -[default4]:[2022-09-10 08:57:17,866] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_19_model_states.pt. -[default0]:[2022-09-10 08:57:17,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_54-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,887] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_52_model_states.pt... -[default0]:[2022-09-10 08:57:17,890] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_52_model_states.pt. -[default4]:[2022-09-10 08:57:17,881] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_67-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:17,881] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_65_model_states.pt... -[default4]:[2022-09-10 08:57:17,898] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_65_model_states.pt. -[default0]:[2022-09-10 08:57:17,807] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_58-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,808] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_56_model_states.pt... -[default0]:[2022-09-10 08:57:17,818] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_56_model_states.pt. -[default4]:[2022-09-10 08:57:17,851] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_43-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:17,851] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_41_model_states.pt... -[default4]:[2022-09-10 08:57:17,876] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_41_model_states.pt. -[default0]:[2022-09-10 08:57:17,919] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_06-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,919] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_04_model_states.pt... -[default4]:[2022-09-10 08:57:17,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_17-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:17,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_15_model_states.pt... -[default4]:[2022-09-10 08:57:17,864] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_15_model_states.pt. -[default4]:[2022-09-10 08:57:17,920] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_27-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:17,920] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_25_model_states.pt... -[default4]:[2022-09-10 08:57:17,946] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_25_model_states.pt. -[default0]:[2022-09-10 08:57:17,892] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_08-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,892] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_06_model_states.pt... -[default0]:[2022-09-10 08:57:17,899] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_06_model_states.pt. -[default4]:[2022-09-10 08:57:17,967] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_33-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:17,968] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_31_model_states.pt... -[default4]:[2022-09-10 08:57:17,972] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_31_model_states.pt. -[default0]:[2022-09-10 08:57:17,971] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_22-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,971] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_20_model_states.pt... -[default4]:[2022-09-10 08:57:17,974] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_49-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:17,974] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_47_model_states.pt... -[default4]:[2022-09-10 08:57:18,001] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_47_model_states.pt. -[default0]:[2022-09-10 08:57:17,914] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_42-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,914] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_40_model_states.pt... -[default0]:[2022-09-10 08:57:17,918] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_40_model_states.pt. -[default4]:[2022-09-10 08:57:18,014] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_07-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,014] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_05_model_states.pt... -[default4]:[2022-09-10 08:57:18,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_05_model_states.pt. -[default0]:[2022-09-10 08:57:17,966] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_20-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:17,967] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_18_model_states.pt... -[default0]:[2022-09-10 08:57:17,969] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_18_model_states.pt. -[default4]:[2022-09-10 08:57:18,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_41-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,005] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_39_model_states.pt... -[default4]:[2022-09-10 08:57:18,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_39_model_states.pt. -[default0]:[2022-09-10 08:57:17,953] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_04_model_states.pt. -[default4]:[2022-09-10 08:57:18,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_15-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,017] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_13_model_states.pt... -[default4]:[2022-09-10 08:57:18,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_13_model_states.pt. -[default0]:[2022-09-10 08:57:18,032] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_14-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,032] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_12_model_states.pt... -[default0]:[2022-09-10 08:57:18,035] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_12_model_states.pt. -[default4]:[2022-09-10 08:57:18,034] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_69-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,034] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_67_model_states.pt... -[default4]:[2022-09-10 08:57:18,037] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_67_model_states.pt. -[default0]:[2022-09-10 08:57:18,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_64-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,029] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_62_model_states.pt... -[default0]:[2022-09-10 08:57:18,031] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_62_model_states.pt. -[default4]:[2022-09-10 08:57:18,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_39-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,036] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_37_model_states.pt... -[default4]:[2022-09-10 08:57:18,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_37_model_states.pt. -[default4]:[2022-09-10 08:57:18,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_09-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,013] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_07_model_states.pt... -[default4]:[2022-09-10 08:57:18,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_07_model_states.pt. -[default0]:[2022-09-10 08:57:18,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_20_model_states.pt. -[default4]:[2022-09-10 08:57:18,032] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_59-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,033] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_57_model_states.pt... -[default4]:[2022-09-10 08:57:18,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_57_model_states.pt. -[default4]:[2022-09-10 08:57:18,080] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_23-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,081] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_21_model_states.pt... -[default4]:[2022-09-10 08:57:18,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_21_model_states.pt. -[default0]:[2022-09-10 08:57:18,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_24-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,071] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_22_model_states.pt... -[default0]:[2022-09-10 08:57:18,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_22_model_states.pt. -[default4]:[2022-09-10 08:57:18,091] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_55-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,091] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_53_model_states.pt... -[default4]:[2022-09-10 08:57:18,093] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_53_model_states.pt. -[default0]:[2022-09-10 08:57:18,098] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_50-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,099] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_48_model_states.pt... -[default0]:[2022-09-10 08:57:18,105] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_48_model_states.pt. -[default0]:[2022-09-10 08:57:18,109] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_26-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,109] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_24_model_states.pt... -[default0]:[2022-09-10 08:57:18,119] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_24_model_states.pt. -[default4]:[2022-09-10 08:57:18,151] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_35-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,152] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_33_model_states.pt... -[default4]:[2022-09-10 08:57:18,161] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_33_model_states.pt. -[default0]:[2022-09-10 08:57:18,155] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_48-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,155] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_46_model_states.pt... -[default0]:[2022-09-10 08:57:18,159] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_46_model_states.pt. -[default0]:[2022-09-10 08:57:18,148] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_70-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,148] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_68_model_states.pt... -[default0]:[2022-09-10 08:57:18,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_68_model_states.pt. -[default4]:[2022-09-10 08:57:18,109] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_61-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,109] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_59_model_states.pt... -[default4]:[2022-09-10 08:57:18,119] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_59_model_states.pt. -[default0]:[2022-09-10 08:57:18,139] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_12-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,139] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_10_model_states.pt... -[default0]:[2022-09-10 08:57:18,148] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_10_model_states.pt. -[default0]:[2022-09-10 08:57:18,182] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_44-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,182] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_42_model_states.pt... -[default0]:[2022-09-10 08:57:18,185] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_42_model_states.pt. -[default0]:[2022-09-10 08:57:18,205] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_52-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,206] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_50_model_states.pt... -[default0]:[2022-09-10 08:57:18,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_46-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_44_model_states.pt... -[default0]:[2022-09-10 08:57:18,200] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_44_model_states.pt. -[default4]:[2022-09-10 08:57:18,217] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_47-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,218] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_45_model_states.pt... -[default4]:[2022-09-10 08:57:18,220] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_45_model_states.pt. -[default4]:[2022-09-10 08:57:18,211] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_45-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,211] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_43_model_states.pt... -[default4]:[2022-09-10 08:57:18,213] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_43_model_states.pt. -[default0]:[2022-09-10 08:57:18,222] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_38-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,222] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_36_model_states.pt... -[default0]:[2022-09-10 08:57:18,225] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_36_model_states.pt. -[default4]:[2022-09-10 08:57:18,229] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_19-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,229] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_17_model_states.pt... -[default4]:[2022-09-10 08:57:18,232] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_17_model_states.pt. -[default0]:[2022-09-10 08:57:18,179] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_18-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,180] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_16_model_states.pt... -[default0]:[2022-09-10 08:57:18,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_16_model_states.pt. -[default0]:[2022-09-10 08:57:18,158] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_28-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,158] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_26_model_states.pt... -[default0]:[2022-09-10 08:57:18,165] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_26_model_states.pt. -[default0]:[2022-09-10 08:57:18,170] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_60-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,171] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_58_model_states.pt... -[default0]:[2022-09-10 08:57:18,174] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_58_model_states.pt. -[default0]:[2022-09-10 08:57:18,232] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_56-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,232] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_54_model_states.pt... -[default0]:[2022-09-10 08:57:18,235] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_54_model_states.pt. -[default4]:[2022-09-10 08:57:18,163] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_13-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,164] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_11_model_states.pt... -[default4]:[2022-09-10 08:57:18,166] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_11_model_states.pt. -[default0]:[2022-09-10 08:57:18,182] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_68-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,183] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_66_model_states.pt... -[default0]:[2022-09-10 08:57:18,185] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_66_model_states.pt. -[default4]:[2022-09-10 08:57:18,205] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_71-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,206] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_69_model_states.pt... -[default4]:[2022-09-10 08:57:18,212] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_69_model_states.pt. -[default4]:[2022-09-10 08:57:18,246] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_25-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,246] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_23_model_states.pt... -[default4]:[2022-09-10 08:57:18,248] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_23_model_states.pt. -[default0]:[2022-09-10 08:57:18,225] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_36-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,226] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_34_model_states.pt... -[default0]:[2022-09-10 08:57:18,233] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_34_model_states.pt. -[default0]:[2022-09-10 08:57:18,213] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_50_model_states.pt. -[default4]:[2022-09-10 08:57:18,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_63-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,284] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_61_model_states.pt... -[default4]:[2022-09-10 08:57:18,287] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_61_model_states.pt. -[default4]:[2022-09-10 08:57:18,220] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_53-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,221] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_51_model_states.pt... -[default4]:[2022-09-10 08:57:18,223] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_51_model_states.pt. -[default0]:[2022-09-10 08:57:18,235] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_62-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,236] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_60_model_states.pt... -[default0]:[2022-09-10 08:57:18,242] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_60_model_states.pt. -[default0]:[2022-09-10 08:57:18,265] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_04-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,265] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_02_model_states.pt... -[default0]:[2022-09-10 08:57:18,269] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_02_model_states.pt. -[default4]:[2022-09-10 08:57:18,290] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_51-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,290] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_49_model_states.pt... -[default4]:[2022-09-10 08:57:18,293] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_49_model_states.pt. -[default4]:[2022-09-10 08:57:18,248] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_57-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,249] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_55_model_states.pt... -[default4]:[2022-09-10 08:57:18,251] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_55_model_states.pt. -[default4]:[2022-09-10 08:57:18,292] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_29-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_27_model_states.pt... -[default4]:[2022-09-10 08:57:18,295] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_27_model_states.pt. -[default4]:[2022-09-10 08:57:18,386] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_05-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,386] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_03_model_states.pt... -[default0]:[2022-09-10 08:57:18,316] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_34-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_32_model_states.pt... -[default0]:[2022-09-10 08:57:18,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_32_model_states.pt. -[default4]:[2022-09-10 08:57:18,329] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_37-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,329] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_35_model_states.pt... -[default4]:[2022-09-10 08:57:18,331] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_35_model_states.pt. -[default4]:[2022-09-10 08:57:18,369] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_03-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,369] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_01_model_states.pt... -[default4]:[2022-09-10 08:57:18,373] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_01_model_states.pt. -[default4]:[2022-09-10 08:57:18,388] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_03_model_states.pt. -[default4]:[2022-09-10 08:57:18,490] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_11-model_00-model_states.pt. -[default4]:[2022-09-10 08:57:18,491] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_09_model_states.pt... -[default4]:[2022-09-10 08:57:18,495] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_09_model_states.pt. -[default0]:[2022-09-10 08:57:18,513] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_10-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:18,514] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_08_model_states.pt... -[default0]:[2022-09-10 08:57:18,515] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_08_model_states.pt. -[default0]:[2022-09-10 08:57:19,185] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/layer_01-model_00-model_states.pt. -[default0]:[2022-09-10 08:57:19,185] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_00_model_states.pt -[default0]:[2022-09-10 08:57:19,185] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_00_model_states.pt... -[default0]:[2022-09-10 08:57:19,189] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/mp_rank_00_model_states.pt. -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default6]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default4]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default5]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default1]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default2]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default0]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default3]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default7]:[2022-09-10 08:57:19,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default5]:[2022-09-10 08:57:27,542] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-10 08:57:27,542] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default1]:[2022-09-10 08:57:27,896] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-10 08:57:27,896] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default2]:[2022-09-10 08:57:28,103] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-10 08:57:28,103] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default3]:[2022-09-10 08:57:28,369] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-10 08:57:28,369] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default1]:[2022-09-10 08:57:28,536] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-10 08:57:28,536] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default6]:[2022-09-10 08:57:28,659] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-10 08:57:28,659] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default5]:[2022-09-10 08:57:28,729] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-10 08:57:28,729] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default1]:[2022-09-10 08:57:28,695] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-10 08:57:28,696] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default1]:[2022-09-10 08:57:28,817] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-10 08:57:28,817] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default4]:[2022-09-10 08:57:28,796] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-10 08:57:28,796] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default1]:[2022-09-10 08:57:28,911] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-10 08:57:28,911] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default5]:[2022-09-10 08:57:28,951] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-10 08:57:28,951] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default1]:[2022-09-10 08:57:29,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-10 08:57:29,043] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default7]:[2022-09-10 08:57:29,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-10 08:57:29,005] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default0]:[2022-09-10 08:57:29,026] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-10 08:57:29,026] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default4]:[2022-09-10 08:57:29,158] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-10 08:57:29,158] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default4]:[2022-09-10 08:57:29,199] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-10 08:57:29,200] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default7]:[2022-09-10 08:57:29,274] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-10 08:57:29,274] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default2]:[2022-09-10 08:57:29,241] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-10 08:57:29,242] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default5]:[2022-09-10 08:57:29,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-10 08:57:29,310] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default0]:[2022-09-10 08:57:29,285] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-10 08:57:29,285] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default4]:[2022-09-10 08:57:29,302] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-10 08:57:29,302] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default0]:[2022-09-10 08:57:29,274] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-10 08:57:29,274] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default3]:[2022-09-10 08:57:29,350] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-10 08:57:29,350] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default6]:[2022-09-10 08:57:29,379] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-10 08:57:29,379] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default0]:[2022-09-10 08:57:29,394] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-10 08:57:29,394] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default5]:[2022-09-10 08:57:29,365] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-10 08:57:29,365] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default4]:[2022-09-10 08:57:29,383] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-10 08:57:29,383] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default6]:[2022-09-10 08:57:29,459] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-10 08:57:29,460] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default2]:[2022-09-10 08:57:29,468] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-10 08:57:29,468] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default0]:[2022-09-10 08:57:29,421] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-10 08:57:29,421] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default7]:[2022-09-10 08:57:29,450] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-10 08:57:29,450] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default5]:[2022-09-10 08:57:29,510] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-10 08:57:29,510] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default3]:[2022-09-10 08:57:29,460] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-10 08:57:29,460] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default5]:[2022-09-10 08:57:29,509] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-10 08:57:29,510] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default0]:[2022-09-10 08:57:29,575] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-10 08:57:29,575] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default3]:[2022-09-10 08:57:29,597] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-10 08:57:29,597] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default2]:[2022-09-10 08:57:29,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-10 08:57:29,561] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default2]:[2022-09-10 08:57:29,567] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-10 08:57:29,567] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default3]:[2022-09-10 08:57:29,613] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-10 08:57:29,613] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default3]:[2022-09-10 08:57:29,680] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-10 08:57:29,680] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default1]:[2022-09-10 08:57:29,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-10 08:57:29,700] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default1]:[2022-09-10 08:57:29,738] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-10 08:57:29,738] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default4]:[2022-09-10 08:57:29,827] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-10 08:57:29,827] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default0]:[2022-09-10 08:57:29,810] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-10 08:57:29,810] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default4]:[2022-09-10 08:57:29,799] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-10 08:57:29,799] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default0]:[2022-09-10 08:57:29,827] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-10 08:57:29,827] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default5]:[2022-09-10 08:57:29,891] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-10 08:57:29,891] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default2]:[2022-09-10 08:57:29,886] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-10 08:57:29,887] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default6]:[2022-09-10 08:57:29,881] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-10 08:57:29,881] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default4]:[2022-09-10 08:57:29,935] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-10 08:57:29,935] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default6]:[2022-09-10 08:57:30,001] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-10 08:57:30,001] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default4]:[2022-09-10 08:57:29,965] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-10 08:57:29,965] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default1]:[2022-09-10 08:57:29,959] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-10 08:57:29,960] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default4]:[2022-09-10 08:57:30,033] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-10 08:57:30,033] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default1]:[2022-09-10 08:57:30,061] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-10 08:57:30,061] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default5]:[2022-09-10 08:57:30,058] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-10 08:57:30,058] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default5]:[2022-09-10 08:57:30,125] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-10 08:57:30,126] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default1]:[2022-09-10 08:57:30,176] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-10 08:57:30,176] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default5]:[2022-09-10 08:57:30,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-10 08:57:30,187] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default7]:[2022-09-10 08:57:30,145] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-10 08:57:30,146] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default0]:[2022-09-10 08:57:30,200] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-10 08:57:30,201] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default5]:[2022-09-10 08:57:30,265] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-10 08:57:30,265] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default4]:[2022-09-10 08:57:30,275] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-10 08:57:30,275] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default4]:[2022-09-10 08:57:30,255] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-10 08:57:30,255] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default3]:[2022-09-10 08:57:30,218] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-10 08:57:30,218] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default5]:[2022-09-10 08:57:30,224] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-10 08:57:30,225] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default0]:[2022-09-10 08:57:30,226] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-10 08:57:30,226] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default0]:[2022-09-10 08:57:30,263] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-10 08:57:30,264] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default0]:[2022-09-10 08:57:30,350] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-10 08:57:30,350] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default7]:[2022-09-10 08:57:30,333] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-10 08:57:30,334] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default6]:[2022-09-10 08:57:30,390] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-10 08:57:30,390] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default2]:[2022-09-10 08:57:30,397] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-10 08:57:30,397] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default5]:[2022-09-10 08:57:30,470] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-10 08:57:30,470] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default5]:[2022-09-10 08:57:30,463] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-10 08:57:30,463] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default3]:[2022-09-10 08:57:30,450] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-10 08:57:30,450] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default0]:[2022-09-10 08:57:30,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-10 08:57:30,416] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default2]:[2022-09-10 08:57:30,460] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-10 08:57:30,460] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default7]:[2022-09-10 08:57:30,503] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-10 08:57:30,503] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default1]:[2022-09-10 08:57:30,517] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-10 08:57:30,517] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default5]:[2022-09-10 08:57:30,497] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-10 08:57:30,497] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default3]:[2022-09-10 08:57:30,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-10 08:57:30,595] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default0]:[2022-09-10 08:57:30,590] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-10 08:57:30,590] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default0]:[2022-09-10 08:57:30,605] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-10 08:57:30,605] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default2]:[2022-09-10 08:57:30,628] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-10 08:57:30,628] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default6]:[2022-09-10 08:57:30,566] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-10 08:57:30,566] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default7]:[2022-09-10 08:57:30,589] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-10 08:57:30,590] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default0]:[2022-09-10 08:57:30,604] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-10 08:57:30,604] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default7]:[2022-09-10 08:57:30,611] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-10 08:57:30,611] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default0]:[2022-09-10 08:57:30,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-10 08:57:30,576] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default2]:[2022-09-10 08:57:30,629] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-10 08:57:30,629] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default1]:[2022-09-10 08:57:30,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-10 08:57:30,579] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default6]:[2022-09-10 08:57:30,660] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-10 08:57:30,660] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default5]:[2022-09-10 08:57:30,642] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-10 08:57:30,642] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default6]:[2022-09-10 08:57:30,632] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-10 08:57:30,633] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default6]:[2022-09-10 08:57:30,720] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-10 08:57:30,720] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default3]:[2022-09-10 08:57:30,668] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-10 08:57:30,668] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default7]:[2022-09-10 08:57:30,696] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-10 08:57:30,696] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default1]:[2022-09-10 08:57:30,666] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-10 08:57:30,666] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default6]:[2022-09-10 08:57:30,697] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-10 08:57:30,697] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default3]:[2022-09-10 08:57:30,716] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-10 08:57:30,716] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default7]:[2022-09-10 08:57:30,731] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-10 08:57:30,731] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default7]:[2022-09-10 08:57:30,718] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-10 08:57:30,718] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default7]:[2022-09-10 08:57:30,756] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-10 08:57:30,756] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default2]:[2022-09-10 08:57:30,806] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-10 08:57:30,806] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default4]:[2022-09-10 08:57:30,833] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-10 08:57:30,834] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default3]:[2022-09-10 08:57:30,839] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-10 08:57:30,839] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default1]:[2022-09-10 08:57:30,827] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-10 08:57:30,828] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default2]:[2022-09-10 08:57:30,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-10 08:57:30,784] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default3]:[2022-09-10 08:57:30,869] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-10 08:57:30,869] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default5]:[2022-09-10 08:57:30,817] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-10 08:57:30,817] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default6]:[2022-09-10 08:57:30,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-10 08:57:30,887] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default6]:[2022-09-10 08:57:30,845] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-10 08:57:30,845] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default2]:[2022-09-10 08:57:30,805] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-10 08:57:30,805] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default7]:[2022-09-10 08:57:30,900] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-10 08:57:30,900] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default0]:[2022-09-10 08:57:30,917] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-10 08:57:30,917] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default6]:[2022-09-10 08:57:30,886] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-10 08:57:30,886] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default3]:[2022-09-10 08:57:30,894] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-10 08:57:30,895] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default4]:[2022-09-10 08:57:30,876] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-10 08:57:30,876] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default6]:[2022-09-10 08:57:30,875] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-10 08:57:30,875] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default7]:[2022-09-10 08:57:30,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-10 08:57:30,947] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default2]:[2022-09-10 08:57:30,889] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-10 08:57:30,889] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default3]:[2022-09-10 08:57:30,946] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-10 08:57:30,946] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default7]:[2022-09-10 08:57:30,961] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-10 08:57:30,961] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default6]:[2022-09-10 08:57:30,952] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-10 08:57:30,952] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default4]:[2022-09-10 08:57:30,961] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-10 08:57:30,961] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default6]:[2022-09-10 08:57:30,926] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-10 08:57:30,927] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default5]:[2022-09-10 08:57:31,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-10 08:57:31,021] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default1]:[2022-09-10 08:57:30,984] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-10 08:57:30,985] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default7]:[2022-09-10 08:57:31,031] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-10 08:57:31,031] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default4]:[2022-09-10 08:57:30,984] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-10 08:57:30,984] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default3]:[2022-09-10 08:57:30,969] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-10 08:57:30,969] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default4]:[2022-09-10 08:57:31,035] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-10 08:57:31,035] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default2]:[2022-09-10 08:57:30,984] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-10 08:57:30,985] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default1]:[2022-09-10 08:57:31,026] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-10 08:57:31,026] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default0]:[2022-09-10 08:57:31,050] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-10 08:57:31,050] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default4]:[2022-09-10 08:57:30,992] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-10 08:57:30,992] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default3]:[2022-09-10 08:57:31,030] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-10 08:57:31,030] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default1]:[2022-09-10 08:57:31,073] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-10 08:57:31,073] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default5]:[2022-09-10 08:57:31,103] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-10 08:57:31,103] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default3]:[2022-09-10 08:57:31,120] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-10 08:57:31,120] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default5]:[2022-09-10 08:57:31,052] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-10 08:57:31,052] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default6]:[2022-09-10 08:57:31,162] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-10 08:57:31,162] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default0]:[2022-09-10 08:57:31,080] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-10 08:57:31,080] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default7]:[2022-09-10 08:57:31,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-10 08:57:31,156] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default4]:[2022-09-10 08:57:31,093] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-10 08:57:31,093] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default0]:[2022-09-10 08:57:31,176] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-10 08:57:31,176] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default1]:[2022-09-10 08:57:31,187] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-10 08:57:31,188] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default1]:[2022-09-10 08:57:31,133] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-10 08:57:31,133] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default3]:[2022-09-10 08:57:31,109] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-10 08:57:31,109] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default1]:[2022-09-10 08:57:31,124] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-10 08:57:31,124] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default4]:[2022-09-10 08:57:31,142] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-10 08:57:31,142] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default2]:[2022-09-10 08:57:31,137] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-10 08:57:31,137] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default6]:[2022-09-10 08:57:31,245] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-10 08:57:31,245] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default6]:[2022-09-10 08:57:31,250] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-10 08:57:31,250] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default3]:[2022-09-10 08:57:31,235] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-10 08:57:31,235] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default7]:[2022-09-10 08:57:31,220] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-10 08:57:31,220] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default0]:[2022-09-10 08:57:31,224] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-10 08:57:31,224] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default7]:[2022-09-10 08:57:31,312] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-10 08:57:31,312] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default2]:[2022-09-10 08:57:31,280] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-10 08:57:31,280] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default5]:[2022-09-10 08:57:31,265] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-10 08:57:31,266] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default1]:[2022-09-10 08:57:31,243] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-10 08:57:31,244] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default6]:[2022-09-10 08:57:31,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-10 08:57:31,305] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default5]:[2022-09-10 08:57:31,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-10 08:57:31,262] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default3]:[2022-09-10 08:57:31,280] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-10 08:57:31,280] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default2]:[2022-09-10 08:57:31,342] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-10 08:57:31,342] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default5]:[2022-09-10 08:57:31,285] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-10 08:57:31,286] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default1]:[2022-09-10 08:57:31,348] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-10 08:57:31,348] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default1]:[2022-09-10 08:57:31,370] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-10 08:57:31,370] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default2]:[2022-09-10 08:57:31,415] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-10 08:57:31,416] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default4]:[2022-09-10 08:57:31,429] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-10 08:57:31,429] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default0]:[2022-09-10 08:57:31,427] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-10 08:57:31,427] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default7]:[2022-09-10 08:57:31,471] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-10 08:57:31,471] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default7]:[2022-09-10 08:57:31,418] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-10 08:57:31,418] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default4]:[2022-09-10 08:57:31,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-10 08:57:31,540] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default7]:[2022-09-10 08:57:31,529] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-10 08:57:31,529] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default2]:[2022-09-10 08:57:31,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-10 08:57:31,528] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default4]:[2022-09-10 08:57:31,529] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-10 08:57:31,529] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default4]:[2022-09-10 08:57:31,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-10 08:57:31,528] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default3]:[2022-09-10 08:57:31,581] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-10 08:57:31,581] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default6]:[2022-09-10 08:57:31,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-10 08:57:31,636] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default3]:[2022-09-10 08:57:31,664] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-10 08:57:31,665] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default1]:[2022-09-10 08:57:31,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-10 08:57:31,672] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default7]:[2022-09-10 08:57:31,671] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-10 08:57:31,671] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default6]:[2022-09-10 08:57:31,690] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-10 08:57:31,690] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default3]:[2022-09-10 08:57:31,608] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-10 08:57:31,609] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default5]:[2022-09-10 08:57:31,674] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-10 08:57:31,675] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default2]:[2022-09-10 08:57:31,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-10 08:57:31,675] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default1]:[2022-09-10 08:57:31,645] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-10 08:57:31,645] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default2]:[2022-09-10 08:57:31,663] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-10 08:57:31,663] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default1]:[2022-09-10 08:57:31,731] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-10 08:57:31,731] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default7]:[2022-09-10 08:57:31,740] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-10 08:57:31,740] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default2]:[2022-09-10 08:57:31,767] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-10 08:57:31,767] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default4]:[2022-09-10 08:57:31,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-10 08:57:31,775] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default5]:[2022-09-10 08:57:31,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-10 08:57:31,820] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default6]:[2022-09-10 08:57:31,815] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-10 08:57:31,815] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default5]:[2022-09-10 08:57:31,819] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-10 08:57:31,820] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default4]:[2022-09-10 08:57:31,909] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-10 08:57:31,909] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default6]:[2022-09-10 08:57:31,925] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-10 08:57:31,925] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default0]:[2022-09-10 08:57:31,898] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-10 08:57:31,898] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default0]:[2022-09-10 08:57:31,903] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-10 08:57:31,903] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default7]:[2022-09-10 08:57:31,914] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-10 08:57:31,914] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default2]:[2022-09-10 08:57:31,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-10 08:57:31,948] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default4]:[2022-09-10 08:57:32,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-10 08:57:32,025] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default7]:[2022-09-10 08:57:32,014] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-10 08:57:32,014] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default2]:[2022-09-10 08:57:32,154] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-10 08:57:32,154] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default7]:[2022-09-10 08:57:32,194] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-10 08:57:32,195] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default0]:[2022-09-10 08:57:32,173] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-10 08:57:32,173] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default0]:[2022-09-10 08:57:32,219] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-10 08:57:32,219] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default3]:[2022-09-10 08:57:32,234] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-10 08:57:32,234] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default4]:[2022-09-10 08:57:32,241] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-10 08:57:32,241] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default2]:[2022-09-10 08:57:32,216] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-10 08:57:32,217] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default1]:[2022-09-10 08:57:32,309] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-10 08:57:32,309] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default7]:[2022-09-10 08:57:32,282] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-10 08:57:32,282] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default3]:[2022-09-10 08:57:32,290] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-10 08:57:32,290] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default0]:[2022-09-10 08:57:32,324] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-10 08:57:32,324] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default7]:[2022-09-10 08:57:32,409] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-10 08:57:32,410] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default3]:[2022-09-10 08:57:32,390] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-10 08:57:32,390] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default5]:[2022-09-10 08:57:32,547] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-10 08:57:32,547] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default1]:[2022-09-10 08:57:32,619] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-10 08:57:32,619] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default5]:[2022-09-10 08:57:32,598] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-10 08:57:32,598] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default2]:[2022-09-10 08:57:32,712] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-10 08:57:32,712] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default3]:[2022-09-10 08:57:32,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-10 08:57:32,757] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default3]:[2022-09-10 08:57:32,802] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-10 08:57:32,802] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default2]:[2022-09-10 08:57:32,884] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-10 08:57:32,884] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default4]:[2022-09-10 08:57:33,095] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-10 08:57:33,095] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default2]:[2022-09-10 08:57:33,055] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-10 08:57:33,056] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default6]:[2022-09-10 08:57:33,264] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-10 08:57:33,265] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default6]:[2022-09-10 08:57:33,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-10 08:57:33,276] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default5]:[2022-09-10 08:57:33,244] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-10 08:57:33,244] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default4]:[2022-09-10 08:57:33,341] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-10 08:57:33,341] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default3]:[2022-09-10 08:57:33,273] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-10 08:57:33,274] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default0]:[2022-09-10 08:57:33,447] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-10 08:57:33,448] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default2]:[2022-09-10 08:57:33,403] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-10 08:57:33,403] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default6]:[2022-09-10 08:57:33,508] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-10 08:57:33,509] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default5]:[2022-09-10 08:57:33,507] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-10 08:57:33,507] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default6]:[2022-09-10 08:57:33,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-10 08:57:33,576] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default3]:[2022-09-10 08:57:33,608] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-10 08:57:33,608] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default6]:[2022-09-10 08:57:33,806] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-10 08:57:33,806] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default4]:[2022-09-10 08:57:33,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-10 08:57:33,932] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default3]:[2022-09-10 08:57:34,405] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-10 08:57:34,405] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default7]:[2022-09-10 08:57:34,403] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-10 08:57:34,403] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default3]:[2022-09-10 08:57:34,479] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-10 08:57:34,479] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default2]:[2022-09-10 08:57:34,611] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-10 08:57:34,612] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default1]:[2022-09-10 08:57:34,676] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-10 08:57:34,676] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default5]:[2022-09-10 08:57:34,684] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-10 08:57:34,684] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default4]:[2022-09-10 08:57:34,791] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-10 08:57:34,791] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default2]:[2022-09-10 08:57:34,892] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-10 08:57:34,892] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default5]:[2022-09-10 08:57:34,924] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-10 08:57:34,924] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default2]:[2022-09-10 08:57:34,949] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-10 08:57:34,950] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default0]:[2022-09-10 08:57:35,118] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-10 08:57:35,122] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default0]:[2022-09-10 08:57:35,053] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-10 08:57:35,053] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default6]:[2022-09-10 08:57:35,162] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-10 08:57:35,162] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default3]:[2022-09-10 08:57:35,201] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-10 08:57:35,201] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default6]:[2022-09-10 08:57:35,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-10 08:57:35,262] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default7]:[2022-09-10 08:57:35,337] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-10 08:57:35,337] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default7]:[2022-09-10 08:57:35,377] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-10 08:57:35,377] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default1]:[2022-09-10 08:57:35,471] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-10 08:57:35,471] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default1]:[2022-09-10 08:57:35,447] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-10 08:57:35,447] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default0]:[2022-09-10 08:57:35,603] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-10 08:57:35,603] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default6]:[2022-09-10 08:57:35,726] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-10 08:57:35,727] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default7]:[2022-09-10 08:57:36,564] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-10 08:57:36,564] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default4]:[2022-09-10 08:57:36,734] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-10 08:57:36,735] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default6]:[2022-09-10 08:57:36,847] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-10 08:57:36,847] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default5]:[2022-09-10 08:57:37,031] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-10 08:57:37,031] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default0]:[2022-09-10 08:57:37,139] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-10 08:57:37,139] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default1]:[2022-09-10 08:57:37,192] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-10 08:57:37,192] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default7]:[2022-09-10 08:57:37,462] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-10 08:57:37,462] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default4]:[2022-09-10 08:57:37,555] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-10 08:57:37,555] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default2]:[2022-09-10 08:57:37,626] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-10 08:57:37,626] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default0]:[2022-09-10 08:57:38,485] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-10 08:57:38,485] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default1]:[2022-09-10 08:57:38,476] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-10 08:57:38,476] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default3]:[2022-09-10 08:57:39,390] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-10 08:57:39,390] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default0]:[2022-09-10 08:57:41,127] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-10 08:57:41,127] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default2]:[2022-09-10 08:57:41,174] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-10 08:57:41,174] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default2]:[2022-09-10 08:57:42,191] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-10 08:57:42,191] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default0]:[2022-09-10 08:57:42,191] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-10 08:57:42,191] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default3]:[2022-09-10 08:57:42,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-10 08:57:42,204] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default1]:[2022-09-10 08:57:42,212] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-10 08:57:42,213] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default1]:[2022-09-10 08:57:43,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-10 08:57:43,831] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default5]:[2022-09-10 08:57:44,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-10 08:57:44,851] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default4]:[2022-09-10 08:57:46,967] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-10 08:57:46,967] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default5]:[2022-09-10 08:57:48,778] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-10 08:57:48,778] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default4]:[2022-09-10 08:57:49,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-10 08:57:49,820] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default7]:[2022-09-10 08:57:50,471] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-10 08:57:50,471] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default6]:[2022-09-10 08:57:50,589] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-10 08:57:50,589] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,355] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-10 08:57:51,355] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]: successfully saved checkpoint at iteration 1494 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:time (ms) | save-checkpoint: 37232.00 -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,404] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-10 08:57:51,404] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1494/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default4]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default5]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default6]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default3]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default1]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default0]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default2]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]:[2022-09-10 08:57:51,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1494 is ready now! -[default7]: iteration 1495/ 3100 | consumed samples: 3061760 | consumed tokens: 6270484480 | elapsed time per iteration (s): 179.04 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.036089E-01 | grad norm: 0.434 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.439 | TFLOPs: 116.77 | -[default7]: iteration 1496/ 3100 | consumed samples: 3063808 | consumed tokens: 6274678784 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.875469E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1497/ 3100 | consumed samples: 3065856 | consumed tokens: 6278873088 | elapsed time per iteration (s): 141.27 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.154979E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.497 | TFLOPs: 147.99 | -[default7]: iteration 1498/ 3100 | consumed samples: 3067904 | consumed tokens: 6283067392 | elapsed time per iteration (s): 140.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.051049E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.570 | TFLOPs: 148.74 | -[default7]: iteration 1499/ 3100 | consumed samples: 3069952 | consumed tokens: 6287261696 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.923226E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 1500/ 3100 | consumed samples: 3072000 | consumed tokens: 6291456000 | elapsed time per iteration (s): 141.06 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.058984E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.519 | TFLOPs: 148.22 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]:validation_pretraining loss at iteration 1500 | lm loss value: 2.397708E+00 | lm loss PPL: 1.099794E+01 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]: iteration 1501/ 3100 | consumed samples: 3074048 | consumed tokens: 6295650304 | elapsed time per iteration (s): 183.26 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.085795E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.176 | TFLOPs: 114.09 | -[default7]: iteration 1502/ 3100 | consumed samples: 3076096 | consumed tokens: 6299844608 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.037547E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 1503/ 3100 | consumed samples: 3078144 | consumed tokens: 6304038912 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.116954E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.35 | -[default7]: iteration 1504/ 3100 | consumed samples: 3080192 | consumed tokens: 6308233216 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.043283E-01 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.31 | -[default7]: iteration 1505/ 3100 | consumed samples: 3082240 | consumed tokens: 6312427520 | elapsed time per iteration (s): 141.30 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.935923E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.494 | TFLOPs: 147.96 | -[default7]: iteration 1506/ 3100 | consumed samples: 3084288 | consumed tokens: 6316621824 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.915443E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 1507/ 3100 | consumed samples: 3086336 | consumed tokens: 6320816128 | elapsed time per iteration (s): 141.09 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.034517E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.516 | TFLOPs: 148.18 | -[default7]: iteration 1508/ 3100 | consumed samples: 3088384 | consumed tokens: 6325010432 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.080597E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.72 | -[default7]: iteration 1509/ 3100 | consumed samples: 3090432 | consumed tokens: 6329204736 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.059927E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 1510/ 3100 | consumed samples: 3092480 | consumed tokens: 6333399040 | elapsed time per iteration (s): 140.11 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.950380E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.617 | TFLOPs: 149.21 | -[default7]: iteration 1511/ 3100 | consumed samples: 3094528 | consumed tokens: 6337593344 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.923320E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 1512/ 3100 | consumed samples: 3096576 | consumed tokens: 6341787648 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.961558E-01 | grad norm: 0.762 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 1513/ 3100 | consumed samples: 3098624 | consumed tokens: 6345981952 | elapsed time per iteration (s): 141.27 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.971983E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.497 | TFLOPs: 147.99 | -[default7]: iteration 1514/ 3100 | consumed samples: 3100672 | consumed tokens: 6350176256 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.088251E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 1515/ 3100 | consumed samples: 3102720 | consumed tokens: 6354370560 | elapsed time per iteration (s): 141.14 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.018130E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.510 | TFLOPs: 148.12 | -[default7]: iteration 1516/ 3100 | consumed samples: 3104768 | consumed tokens: 6358564864 | elapsed time per iteration (s): 140.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.845412E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.530 | TFLOPs: 148.33 | -[default7]: iteration 1517/ 3100 | consumed samples: 3106816 | consumed tokens: 6362759168 | elapsed time per iteration (s): 143.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.970388E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.264 | TFLOPs: 145.62 | -[default7]: iteration 1518/ 3100 | consumed samples: 3108864 | consumed tokens: 6366953472 | elapsed time per iteration (s): 141.34 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.877880E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.490 | TFLOPs: 147.92 | -[default7]: iteration 1519/ 3100 | consumed samples: 3110912 | consumed tokens: 6371147776 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.947140E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 1520/ 3100 | consumed samples: 3112960 | consumed tokens: 6375342080 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.080019E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.427 | TFLOPs: 147.27 | -[default7]: iteration 1521/ 3100 | consumed samples: 3115008 | consumed tokens: 6379536384 | elapsed time per iteration (s): 140.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.937632E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.531 | TFLOPs: 148.34 | -[default7]: iteration 1522/ 3100 | consumed samples: 3117056 | consumed tokens: 6383730688 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.956881E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1523/ 3100 | consumed samples: 3119104 | consumed tokens: 6387924992 | elapsed time per iteration (s): 141.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.087603E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.488 | TFLOPs: 147.90 | -[default7]: iteration 1524/ 3100 | consumed samples: 3121152 | consumed tokens: 6392119296 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.002337E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1525/ 3100 | consumed samples: 3123200 | consumed tokens: 6396313600 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.203078E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 1526/ 3100 | consumed samples: 3125248 | consumed tokens: 6400507904 | elapsed time per iteration (s): 140.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.955321E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.554 | TFLOPs: 148.57 | -[default7]: iteration 1527/ 3100 | consumed samples: 3127296 | consumed tokens: 6404702208 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.898937E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 1528/ 3100 | consumed samples: 3129344 | consumed tokens: 6408896512 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.983680E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1529/ 3100 | consumed samples: 3131392 | consumed tokens: 6413090816 | elapsed time per iteration (s): 141.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.092475E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.429 | TFLOPs: 147.30 | -[default7]: iteration 1530/ 3100 | consumed samples: 3133440 | consumed tokens: 6417285120 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.888950E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 1531/ 3100 | consumed samples: 3135488 | consumed tokens: 6421479424 | elapsed time per iteration (s): 141.13 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.984428E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.512 | TFLOPs: 148.14 | -[default7]: iteration 1532/ 3100 | consumed samples: 3137536 | consumed tokens: 6425673728 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.911644E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1533/ 3100 | consumed samples: 3139584 | consumed tokens: 6429868032 | elapsed time per iteration (s): 141.19 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.072349E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.505 | TFLOPs: 148.08 | -[default7]: iteration 1534/ 3100 | consumed samples: 3141632 | consumed tokens: 6434062336 | elapsed time per iteration (s): 141.17 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.963858E-01 | grad norm: 0.486 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.507 | TFLOPs: 148.10 | -[default7]: iteration 1535/ 3100 | consumed samples: 3143680 | consumed tokens: 6438256640 | elapsed time per iteration (s): 142.01 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.879750E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.422 | TFLOPs: 147.22 | -[default7]: iteration 1536/ 3100 | consumed samples: 3145728 | consumed tokens: 6442450944 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.947713E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1537/ 3100 | consumed samples: 3147776 | consumed tokens: 6446645248 | elapsed time per iteration (s): 140.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.935852E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.576 | TFLOPs: 148.79 | -[default7]: iteration 1538/ 3100 | consumed samples: 3149824 | consumed tokens: 6450839552 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.942911E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1539/ 3100 | consumed samples: 3151872 | consumed tokens: 6455033856 | elapsed time per iteration (s): 141.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.934717E-01 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.491 | TFLOPs: 147.93 | -[default7]: iteration 1540/ 3100 | consumed samples: 3153920 | consumed tokens: 6459228160 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.862406E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1541/ 3100 | consumed samples: 3155968 | consumed tokens: 6463422464 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.019051E-01 | grad norm: 0.370 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 1542/ 3100 | consumed samples: 3158016 | consumed tokens: 6467616768 | elapsed time per iteration (s): 141.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.047438E-01 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.429 | TFLOPs: 147.30 | -[default7]: iteration 1543/ 3100 | consumed samples: 3160064 | consumed tokens: 6471811072 | elapsed time per iteration (s): 140.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.045439E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.586 | TFLOPs: 148.90 | -[default7]: iteration 1544/ 3100 | consumed samples: 3162112 | consumed tokens: 6476005376 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.066552E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1545/ 3100 | consumed samples: 3164160 | consumed tokens: 6480199680 | elapsed time per iteration (s): 141.15 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.905968E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.509 | TFLOPs: 148.12 | -[default7]: iteration 1546/ 3100 | consumed samples: 3166208 | consumed tokens: 6484393984 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.794588E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1547/ 3100 | consumed samples: 3168256 | consumed tokens: 6488588288 | elapsed time per iteration (s): 141.29 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.907004E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.495 | TFLOPs: 147.97 | -[default7]: iteration 1548/ 3100 | consumed samples: 3170304 | consumed tokens: 6492782592 | elapsed time per iteration (s): 140.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.856772E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.590 | TFLOPs: 148.94 | -[default7]: iteration 1549/ 3100 | consumed samples: 3172352 | consumed tokens: 6496976896 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.900398E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 1550/ 3100 | consumed samples: 3174400 | consumed tokens: 6501171200 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.868402E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1551/ 3100 | consumed samples: 3176448 | consumed tokens: 6505365504 | elapsed time per iteration (s): 141.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.947855E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.523 | TFLOPs: 148.25 | -[default7]: iteration 1552/ 3100 | consumed samples: 3178496 | consumed tokens: 6509559808 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.025739E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1553/ 3100 | consumed samples: 3180544 | consumed tokens: 6513754112 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.856191E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1554/ 3100 | consumed samples: 3182592 | consumed tokens: 6517948416 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.858393E-01 | grad norm: 0.365 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 1555/ 3100 | consumed samples: 3184640 | consumed tokens: 6522142720 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.030937E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.425 | TFLOPs: 147.26 | -[default7]: iteration 1556/ 3100 | consumed samples: 3186688 | consumed tokens: 6526337024 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.882904E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1557/ 3100 | consumed samples: 3188736 | consumed tokens: 6530531328 | elapsed time per iteration (s): 141.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.888364E-01 | grad norm: 0.423 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.429 | TFLOPs: 147.30 | -[default7]: iteration 1558/ 3100 | consumed samples: 3190784 | consumed tokens: 6534725632 | elapsed time per iteration (s): 141.28 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.986563E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.496 | TFLOPs: 147.98 | -[default7]: iteration 1559/ 3100 | consumed samples: 3192832 | consumed tokens: 6538919936 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.999008E-01 | grad norm: 0.370 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 1560/ 3100 | consumed samples: 3194880 | consumed tokens: 6543114240 | elapsed time per iteration (s): 140.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.013357E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.555 | TFLOPs: 148.59 | -[default7]: iteration 1561/ 3100 | consumed samples: 3196928 | consumed tokens: 6547308544 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.985680E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1562/ 3100 | consumed samples: 3198976 | consumed tokens: 6551502848 | elapsed time per iteration (s): 140.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.961131E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.565 | TFLOPs: 148.69 | -[default7]: iteration 1563/ 3100 | consumed samples: 3201024 | consumed tokens: 6555697152 | elapsed time per iteration (s): 140.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.087601E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.583 | TFLOPs: 148.87 | -[default7]: iteration 1564/ 3100 | consumed samples: 3203072 | consumed tokens: 6559891456 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.960435E-01 | grad norm: 0.714 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 1565/ 3100 | consumed samples: 3205120 | consumed tokens: 6564085760 | elapsed time per iteration (s): 141.32 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.889034E-01 | grad norm: 4.167 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.492 | TFLOPs: 147.94 | -[default7]: iteration 1566/ 3100 | consumed samples: 3207168 | consumed tokens: 6568280064 | elapsed time per iteration (s): 141.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.971373E-01 | grad norm: 2.421 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.491 | TFLOPs: 147.93 | -[default7]: iteration 1567/ 3100 | consumed samples: 3209216 | consumed tokens: 6572474368 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.973309E-01 | grad norm: 1.898 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1568/ 3100 | consumed samples: 3211264 | consumed tokens: 6576668672 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.945873E-01 | grad norm: 4.817 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 1569/ 3100 | consumed samples: 3213312 | consumed tokens: 6580862976 | elapsed time per iteration (s): 140.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.788658E-01 | grad norm: 0.518 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.562 | TFLOPs: 148.66 | -[default7]: iteration 1570/ 3100 | consumed samples: 3215360 | consumed tokens: 6585057280 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.018180E-01 | grad norm: 0.533 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.84 | -[default7]: iteration 1571/ 3100 | consumed samples: 3217408 | consumed tokens: 6589251584 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.749153E-01 | grad norm: 1.043 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1572/ 3100 | consumed samples: 3219456 | consumed tokens: 6593445888 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.051701E-01 | grad norm: 0.856 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 1573/ 3100 | consumed samples: 3221504 | consumed tokens: 6597640192 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.862591E-01 | grad norm: 0.588 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.84 | -[default7]: iteration 1574/ 3100 | consumed samples: 3223552 | consumed tokens: 6601834496 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.938465E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.499 | TFLOPs: 148.01 | -[default7]: iteration 1575/ 3100 | consumed samples: 3225600 | consumed tokens: 6606028800 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.935199E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 1576/ 3100 | consumed samples: 3227648 | consumed tokens: 6610223104 | elapsed time per iteration (s): 140.17 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.982330E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.611 | TFLOPs: 149.15 | -[default7]: iteration 1577/ 3100 | consumed samples: 3229696 | consumed tokens: 6614417408 | elapsed time per iteration (s): 140.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.917587E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.583 | TFLOPs: 148.87 | -[default7]: iteration 1578/ 3100 | consumed samples: 3231744 | consumed tokens: 6618611712 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.909617E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1579/ 3100 | consumed samples: 3233792 | consumed tokens: 6622806016 | elapsed time per iteration (s): 141.28 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.820617E-01 | grad norm: 0.954 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.496 | TFLOPs: 147.98 | -[default7]: iteration 1580/ 3100 | consumed samples: 3235840 | consumed tokens: 6627000320 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.938654E-01 | grad norm: 0.413 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 1581/ 3100 | consumed samples: 3237888 | consumed tokens: 6631194624 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.954760E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 1582/ 3100 | consumed samples: 3239936 | consumed tokens: 6635388928 | elapsed time per iteration (s): 141.12 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.867344E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.512 | TFLOPs: 148.14 | -[default7]: iteration 1583/ 3100 | consumed samples: 3241984 | consumed tokens: 6639583232 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.770635E-01 | grad norm: 0.814 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 1584/ 3100 | consumed samples: 3244032 | consumed tokens: 6643777536 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.732286E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.27 | -[default7]: iteration 1585/ 3100 | consumed samples: 3246080 | consumed tokens: 6647971840 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.945294E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1586/ 3100 | consumed samples: 3248128 | consumed tokens: 6652166144 | elapsed time per iteration (s): 141.23 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.023096E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.501 | TFLOPs: 148.03 | -[default7]: iteration 1587/ 3100 | consumed samples: 3250176 | consumed tokens: 6656360448 | elapsed time per iteration (s): 141.22 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.012139E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.502 | TFLOPs: 148.04 | -[default7]: iteration 1588/ 3100 | consumed samples: 3252224 | consumed tokens: 6660554752 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.791851E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1589/ 3100 | consumed samples: 3254272 | consumed tokens: 6664749056 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.969808E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1590/ 3100 | consumed samples: 3256320 | consumed tokens: 6668943360 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.842752E-01 | grad norm: 0.353 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1591/ 3100 | consumed samples: 3258368 | consumed tokens: 6673137664 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.832141E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 1592/ 3100 | consumed samples: 3260416 | consumed tokens: 6677331968 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.069894E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 1593/ 3100 | consumed samples: 3262464 | consumed tokens: 6681526272 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.900695E-01 | grad norm: 0.350 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1594/ 3100 | consumed samples: 3264512 | consumed tokens: 6685720576 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.994181E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1595/ 3100 | consumed samples: 3266560 | consumed tokens: 6689914880 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.902001E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 1596/ 3100 | consumed samples: 3268608 | consumed tokens: 6694109184 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.898678E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1597/ 3100 | consumed samples: 3270656 | consumed tokens: 6698303488 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.848403E-01 | grad norm: 0.661 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.582 | TFLOPs: 148.86 | -[default7]: iteration 1598/ 3100 | consumed samples: 3272704 | consumed tokens: 6702497792 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.884606E-01 | grad norm: 0.813 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 1599/ 3100 | consumed samples: 3274752 | consumed tokens: 6706692096 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.864274E-01 | grad norm: 0.382 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1600/ 3100 | consumed samples: 3276800 | consumed tokens: 6710886400 | elapsed time per iteration (s): 140.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.897903E-01 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.584 | TFLOPs: 148.88 | -[default7]: iteration 1601/ 3100 | consumed samples: 3278848 | consumed tokens: 6715080704 | elapsed time per iteration (s): 141.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.046377E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.484 | TFLOPs: 147.85 | -[default7]: iteration 1602/ 3100 | consumed samples: 3280896 | consumed tokens: 6719275008 | elapsed time per iteration (s): 141.27 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.927018E-01 | grad norm: 0.443 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.498 | TFLOPs: 148.00 | -[default7]: iteration 1603/ 3100 | consumed samples: 3282944 | consumed tokens: 6723469312 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.878960E-01 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1604/ 3100 | consumed samples: 3284992 | consumed tokens: 6727663616 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.786126E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.499 | TFLOPs: 148.01 | -[default7]: iteration 1605/ 3100 | consumed samples: 3287040 | consumed tokens: 6731857920 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.699923E-01 | grad norm: 1.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1606/ 3100 | consumed samples: 3289088 | consumed tokens: 6736052224 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.889335E-01 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1607/ 3100 | consumed samples: 3291136 | consumed tokens: 6740246528 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.882914E-01 | grad norm: 2.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 1608/ 3100 | consumed samples: 3293184 | consumed tokens: 6744440832 | elapsed time per iteration (s): 141.98 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.975642E-01 | grad norm: 0.463 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.424 | TFLOPs: 147.25 | -[default7]: iteration 1609/ 3100 | consumed samples: 3295232 | consumed tokens: 6748635136 | elapsed time per iteration (s): 140.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.988117E-01 | grad norm: 8.427 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.584 | TFLOPs: 148.88 | -[default7]: iteration 1610/ 3100 | consumed samples: 3297280 | consumed tokens: 6752829440 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.885686E-01 | grad norm: 39.199 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 1611/ 3100 | consumed samples: 3299328 | consumed tokens: 6757023744 | elapsed time per iteration (s): 141.98 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.992178E-01 | grad norm: 43.320 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.424 | TFLOPs: 147.25 | -[default7]: iteration 1612/ 3100 | consumed samples: 3301376 | consumed tokens: 6761218048 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.904878E-01 | grad norm: 86.499 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 1613/ 3100 | consumed samples: 3303424 | consumed tokens: 6765412352 | elapsed time per iteration (s): 140.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.073308E-01 | grad norm: 4.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.561 | TFLOPs: 148.65 | -[default7]: iteration 1614/ 3100 | consumed samples: 3305472 | consumed tokens: 6769606656 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.022603E-01 | grad norm: 15.439 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1615/ 3100 | consumed samples: 3307520 | consumed tokens: 6773800960 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.972444E-01 | grad norm: 5.203 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1616/ 3100 | consumed samples: 3309568 | consumed tokens: 6777995264 | elapsed time per iteration (s): 140.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.130896E-01 | grad norm: 15.633 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.587 | TFLOPs: 148.91 | -[default7]: iteration 1617/ 3100 | consumed samples: 3311616 | consumed tokens: 6782189568 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.068249E-01 | grad norm: 27.264 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 1618/ 3100 | consumed samples: 3313664 | consumed tokens: 6786383872 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.176780E-01 | grad norm: 21.267 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 1619/ 3100 | consumed samples: 3315712 | consumed tokens: 6790578176 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.328981E-01 | grad norm: 223.695 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 1620/ 3100 | consumed samples: 3317760 | consumed tokens: 6794772480 | elapsed time per iteration (s): 140.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.307235E-01 | grad norm: 147.563 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.561 | TFLOPs: 148.65 | -[default7]: iteration 1621/ 3100 | consumed samples: 3319808 | consumed tokens: 6798966784 | elapsed time per iteration (s): 140.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.253529E-01 | grad norm: 227.987 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.545 | TFLOPs: 148.48 | -[default7]: iteration 1622/ 3100 | consumed samples: 3321856 | consumed tokens: 6803161088 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.715699E-01 | grad norm: 77.467 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.45 | -[default7]: iteration 1623/ 3100 | consumed samples: 3323904 | consumed tokens: 6807355392 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.655295E-01 | grad norm: 17.225 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1624/ 3100 | consumed samples: 3325952 | consumed tokens: 6811549696 | elapsed time per iteration (s): 140.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.465895E-01 | grad norm: 20.889 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.579 | TFLOPs: 148.83 | -[default7]: iteration 1625/ 3100 | consumed samples: 3328000 | consumed tokens: 6815744000 | elapsed time per iteration (s): 141.31 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.365001E-01 | grad norm: 13.672 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.493 | TFLOPs: 147.95 | -[default7]: iteration 1626/ 3100 | consumed samples: 3330048 | consumed tokens: 6819938304 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.305343E-01 | grad norm: 10.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1627/ 3100 | consumed samples: 3332096 | consumed tokens: 6824132608 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.266727E-01 | grad norm: 29.936 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 1628/ 3100 | consumed samples: 3334144 | consumed tokens: 6828326912 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.152754E-01 | grad norm: 12.323 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1629/ 3100 | consumed samples: 3336192 | consumed tokens: 6832521216 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.240834E-01 | grad norm: 8.221 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1630/ 3100 | consumed samples: 3338240 | consumed tokens: 6836715520 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.183907E-01 | grad norm: 36.956 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 1631/ 3100 | consumed samples: 3340288 | consumed tokens: 6840909824 | elapsed time per iteration (s): 140.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.225633E-01 | grad norm: 23.761 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.571 | TFLOPs: 148.75 | -[default7]: iteration 1632/ 3100 | consumed samples: 3342336 | consumed tokens: 6845104128 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.089311E-01 | grad norm: 2.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1633/ 3100 | consumed samples: 3344384 | consumed tokens: 6849298432 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.995868E-01 | grad norm: 2.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1634/ 3100 | consumed samples: 3346432 | consumed tokens: 6853492736 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.081126E-01 | grad norm: 11.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 1635/ 3100 | consumed samples: 3348480 | consumed tokens: 6857687040 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.084715E-01 | grad norm: 3.038 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 1636/ 3100 | consumed samples: 3350528 | consumed tokens: 6861881344 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.068808E-01 | grad norm: 11.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 1637/ 3100 | consumed samples: 3352576 | consumed tokens: 6866075648 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.033093E-01 | grad norm: 4.828 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1638/ 3100 | consumed samples: 3354624 | consumed tokens: 6870269952 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.932042E-01 | grad norm: 8.188 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1639/ 3100 | consumed samples: 3356672 | consumed tokens: 6874464256 | elapsed time per iteration (s): 141.24 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.025972E-01 | grad norm: 2.263 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.500 | TFLOPs: 148.03 | -[default7]: iteration 1640/ 3100 | consumed samples: 3358720 | consumed tokens: 6878658560 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.049854E-01 | grad norm: 32.870 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1641/ 3100 | consumed samples: 3360768 | consumed tokens: 6882852864 | elapsed time per iteration (s): 141.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.842274E-01 | grad norm: 0.900 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.427 | TFLOPs: 147.28 | -[default7]: iteration 1642/ 3100 | consumed samples: 3362816 | consumed tokens: 6887047168 | elapsed time per iteration (s): 140.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.055289E-01 | grad norm: 2.826 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.548 | TFLOPs: 148.51 | -[default7]: iteration 1643/ 3100 | consumed samples: 3364864 | consumed tokens: 6891241472 | elapsed time per iteration (s): 140.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.895004E-01 | grad norm: 2.202 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.543 | TFLOPs: 148.46 | -[default7]: iteration 1644/ 3100 | consumed samples: 3366912 | consumed tokens: 6895435776 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.940083E-01 | grad norm: 1.983 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.430 | TFLOPs: 147.31 | -[default7]: iteration 1645/ 3100 | consumed samples: 3368960 | consumed tokens: 6899630080 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.889396E-01 | grad norm: 2.167 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 1646/ 3100 | consumed samples: 3371008 | consumed tokens: 6903824384 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.725257E-01 | grad norm: 5.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.35 | -[default7]: iteration 1647/ 3100 | consumed samples: 3373056 | consumed tokens: 6908018688 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.052723E-01 | grad norm: 2.622 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 1648/ 3100 | consumed samples: 3375104 | consumed tokens: 6912212992 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.920154E-01 | grad norm: 4.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 1649/ 3100 | consumed samples: 3377152 | consumed tokens: 6916407296 | elapsed time per iteration (s): 141.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.862523E-01 | grad norm: 0.562 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.428 | TFLOPs: 147.29 | -[default7]: iteration 1650/ 3100 | consumed samples: 3379200 | consumed tokens: 6920601600 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.906711E-01 | grad norm: 2.233 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 1651/ 3100 | consumed samples: 3381248 | consumed tokens: 6924795904 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.198016E-01 | grad norm: 6.648 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1652/ 3100 | consumed samples: 3383296 | consumed tokens: 6928990208 | elapsed time per iteration (s): 142.07 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.971157E-01 | grad norm: 2.133 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.415 | TFLOPs: 147.16 | -[default7]: iteration 1653/ 3100 | consumed samples: 3385344 | consumed tokens: 6933184512 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.898010E-01 | grad norm: 5.216 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1654/ 3100 | consumed samples: 3387392 | consumed tokens: 6937378816 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.801828E-01 | grad norm: 3.145 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 1655/ 3100 | consumed samples: 3389440 | consumed tokens: 6941573120 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.962230E-01 | grad norm: 1.023 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 1656/ 3100 | consumed samples: 3391488 | consumed tokens: 6945767424 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.993314E-01 | grad norm: 1.187 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 1657/ 3100 | consumed samples: 3393536 | consumed tokens: 6949961728 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.927168E-01 | grad norm: 0.837 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 1658/ 3100 | consumed samples: 3395584 | consumed tokens: 6954156032 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.963921E-01 | grad norm: 0.512 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1659/ 3100 | consumed samples: 3397632 | consumed tokens: 6958350336 | elapsed time per iteration (s): 141.30 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.874257E-01 | grad norm: 0.739 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.494 | TFLOPs: 147.96 | -[default7]: iteration 1660/ 3100 | consumed samples: 3399680 | consumed tokens: 6962544640 | elapsed time per iteration (s): 140.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.951397E-01 | grad norm: 8.266 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.568 | TFLOPs: 148.72 | -[default7]: iteration 1661/ 3100 | consumed samples: 3401728 | consumed tokens: 6966738944 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.852827E-01 | grad norm: 0.548 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 1662/ 3100 | consumed samples: 3403776 | consumed tokens: 6970933248 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.019207E-01 | grad norm: 0.530 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 1663/ 3100 | consumed samples: 3405824 | consumed tokens: 6975127552 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.754596E-01 | grad norm: 1.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 1664/ 3100 | consumed samples: 3407872 | consumed tokens: 6979321856 | elapsed time per iteration (s): 141.99 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.878451E-01 | grad norm: 0.485 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.424 | TFLOPs: 147.24 | -[default7]: iteration 1665/ 3100 | consumed samples: 3409920 | consumed tokens: 6983516160 | elapsed time per iteration (s): 141.01 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.010982E-01 | grad norm: 0.449 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.524 | TFLOPs: 148.27 | -[default7]: iteration 1666/ 3100 | consumed samples: 3411968 | consumed tokens: 6987710464 | elapsed time per iteration (s): 140.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.942859E-01 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.586 | TFLOPs: 148.90 | -[default7]: iteration 1667/ 3100 | consumed samples: 3414016 | consumed tokens: 6991904768 | elapsed time per iteration (s): 141.06 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.896003E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.518 | TFLOPs: 148.21 | -[default7]: iteration 1668/ 3100 | consumed samples: 3416064 | consumed tokens: 6996099072 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.901633E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 1669/ 3100 | consumed samples: 3418112 | consumed tokens: 7000293376 | elapsed time per iteration (s): 141.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.776709E-01 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.430 | TFLOPs: 147.31 | -[default7]: iteration 1670/ 3100 | consumed samples: 3420160 | consumed tokens: 7004487680 | elapsed time per iteration (s): 141.19 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.833005E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.505 | TFLOPs: 148.07 | -[default7]: iteration 1671/ 3100 | consumed samples: 3422208 | consumed tokens: 7008681984 | elapsed time per iteration (s): 140.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.822912E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.585 | TFLOPs: 148.89 | -[default7]: iteration 1672/ 3100 | consumed samples: 3424256 | consumed tokens: 7012876288 | elapsed time per iteration (s): 141.27 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.951165E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.497 | TFLOPs: 147.99 | -[default7]: iteration 1673/ 3100 | consumed samples: 3426304 | consumed tokens: 7017070592 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.004644E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1674/ 3100 | consumed samples: 3428352 | consumed tokens: 7021264896 | elapsed time per iteration (s): 141.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.984846E-01 | grad norm: 0.447 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.428 | TFLOPs: 147.29 | -[default7]: iteration 1675/ 3100 | consumed samples: 3430400 | consumed tokens: 7025459200 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.667359E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1676/ 3100 | consumed samples: 3432448 | consumed tokens: 7029653504 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.994962E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 1677/ 3100 | consumed samples: 3434496 | consumed tokens: 7033847808 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.022819E-01 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.45 | -[default7]: iteration 1678/ 3100 | consumed samples: 3436544 | consumed tokens: 7038042112 | elapsed time per iteration (s): 140.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.881260E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.578 | TFLOPs: 148.82 | -[default7]: iteration 1679/ 3100 | consumed samples: 3438592 | consumed tokens: 7042236416 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.761642E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.27 | -[default7]: iteration 1680/ 3100 | consumed samples: 3440640 | consumed tokens: 7046430720 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.791622E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 1681/ 3100 | consumed samples: 3442688 | consumed tokens: 7050625024 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.932910E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1682/ 3100 | consumed samples: 3444736 | consumed tokens: 7054819328 | elapsed time per iteration (s): 140.35 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.787491E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.593 | TFLOPs: 148.97 | -[default7]: iteration 1683/ 3100 | consumed samples: 3446784 | consumed tokens: 7059013632 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.849495E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 1684/ 3100 | consumed samples: 3448832 | consumed tokens: 7063207936 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.765725E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.425 | TFLOPs: 147.26 | -[default7]: iteration 1685/ 3100 | consumed samples: 3450880 | consumed tokens: 7067402240 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.892926E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1686/ 3100 | consumed samples: 3452928 | consumed tokens: 7071596544 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.910116E-01 | grad norm: 0.472 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1687/ 3100 | consumed samples: 3454976 | consumed tokens: 7075790848 | elapsed time per iteration (s): 142.06 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.799641E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.417 | TFLOPs: 147.17 | -[default7]: iteration 1688/ 3100 | consumed samples: 3457024 | consumed tokens: 7079985152 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.784121E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 1689/ 3100 | consumed samples: 3459072 | consumed tokens: 7084179456 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.737959E-01 | grad norm: 0.439 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 1690/ 3100 | consumed samples: 3461120 | consumed tokens: 7088373760 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.748554E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.27 | -[default7]: iteration 1691/ 3100 | consumed samples: 3463168 | consumed tokens: 7092568064 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.830307E-01 | grad norm: 0.441 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1692/ 3100 | consumed samples: 3465216 | consumed tokens: 7096762368 | elapsed time per iteration (s): 141.13 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.873695E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.511 | TFLOPs: 148.14 | -[default7]: iteration 1693/ 3100 | consumed samples: 3467264 | consumed tokens: 7100956672 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.812493E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1694/ 3100 | consumed samples: 3469312 | consumed tokens: 7105150976 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.783961E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.44 | -[default7]: iteration 1695/ 3100 | consumed samples: 3471360 | consumed tokens: 7109345280 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.954357E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1696/ 3100 | consumed samples: 3473408 | consumed tokens: 7113539584 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.868187E-01 | grad norm: 0.407 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 1697/ 3100 | consumed samples: 3475456 | consumed tokens: 7117733888 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.808893E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1698/ 3100 | consumed samples: 3477504 | consumed tokens: 7121928192 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.729592E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1699/ 3100 | consumed samples: 3479552 | consumed tokens: 7126122496 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.921756E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 1700/ 3100 | consumed samples: 3481600 | consumed tokens: 7130316800 | elapsed time per iteration (s): 139.93 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.669129E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.635 | TFLOPs: 149.41 | -[default7]: iteration 1701/ 3100 | consumed samples: 3483648 | consumed tokens: 7134511104 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.760561E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1702/ 3100 | consumed samples: 3485696 | consumed tokens: 7138705408 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.748420E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1703/ 3100 | consumed samples: 3487744 | consumed tokens: 7142899712 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.763835E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 1704/ 3100 | consumed samples: 3489792 | consumed tokens: 7147094016 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.738097E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1705/ 3100 | consumed samples: 3491840 | consumed tokens: 7151288320 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.795843E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 1706/ 3100 | consumed samples: 3493888 | consumed tokens: 7155482624 | elapsed time per iteration (s): 140.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.714911E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.574 | TFLOPs: 148.78 | -[default7]: iteration 1707/ 3100 | consumed samples: 3495936 | consumed tokens: 7159676928 | elapsed time per iteration (s): 141.31 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.730215E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.493 | TFLOPs: 147.95 | -[default7]: iteration 1708/ 3100 | consumed samples: 3497984 | consumed tokens: 7163871232 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.925968E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1709/ 3100 | consumed samples: 3500032 | consumed tokens: 7168065536 | elapsed time per iteration (s): 141.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.843829E-01 | grad norm: 0.682 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.491 | TFLOPs: 147.93 | -[default7]: iteration 1710/ 3100 | consumed samples: 3502080 | consumed tokens: 7172259840 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.880594E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.74 | -[default7]: iteration 1711/ 3100 | consumed samples: 3504128 | consumed tokens: 7176454144 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.822475E-01 | grad norm: 0.347 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1712/ 3100 | consumed samples: 3506176 | consumed tokens: 7180648448 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.922587E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1713/ 3100 | consumed samples: 3508224 | consumed tokens: 7184842752 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.783422E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 1714/ 3100 | consumed samples: 3510272 | consumed tokens: 7189037056 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.892945E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 1715/ 3100 | consumed samples: 3512320 | consumed tokens: 7193231360 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.686383E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1716/ 3100 | consumed samples: 3514368 | consumed tokens: 7197425664 | elapsed time per iteration (s): 140.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.934093E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.586 | TFLOPs: 148.90 | -[default7]: iteration 1717/ 3100 | consumed samples: 3516416 | consumed tokens: 7201619968 | elapsed time per iteration (s): 140.18 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.786675E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.610 | TFLOPs: 149.14 | -[default7]: iteration 1718/ 3100 | consumed samples: 3518464 | consumed tokens: 7205814272 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.680856E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1719/ 3100 | consumed samples: 3520512 | consumed tokens: 7210008576 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.711821E-01 | grad norm: 0.382 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1720/ 3100 | consumed samples: 3522560 | consumed tokens: 7214202880 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.625509E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 1721/ 3100 | consumed samples: 3524608 | consumed tokens: 7218397184 | elapsed time per iteration (s): 140.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.652138E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.583 | TFLOPs: 148.87 | -[default7]: iteration 1722/ 3100 | consumed samples: 3526656 | consumed tokens: 7222591488 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.827243E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1723/ 3100 | consumed samples: 3528704 | consumed tokens: 7226785792 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.794704E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.26 | -[default7]: iteration 1724/ 3100 | consumed samples: 3530752 | consumed tokens: 7230980096 | elapsed time per iteration (s): 141.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.676332E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.428 | TFLOPs: 147.29 | -[default7]: iteration 1725/ 3100 | consumed samples: 3532800 | consumed tokens: 7235174400 | elapsed time per iteration (s): 143.28 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.842804E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.294 | TFLOPs: 145.92 | -[default7]: iteration 1726/ 3100 | consumed samples: 3534848 | consumed tokens: 7239368704 | elapsed time per iteration (s): 140.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.816625E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.569 | TFLOPs: 148.73 | -[default7]: iteration 1727/ 3100 | consumed samples: 3536896 | consumed tokens: 7243563008 | elapsed time per iteration (s): 142.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.640328E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.421 | TFLOPs: 147.21 | -[default7]: iteration 1728/ 3100 | consumed samples: 3538944 | consumed tokens: 7247757312 | elapsed time per iteration (s): 140.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.749050E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.561 | TFLOPs: 148.64 | -[default7]: iteration 1729/ 3100 | consumed samples: 3540992 | consumed tokens: 7251951616 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.697922E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 1730/ 3100 | consumed samples: 3543040 | consumed tokens: 7256145920 | elapsed time per iteration (s): 141.06 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.849142E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.519 | TFLOPs: 148.21 | -[default7]: iteration 1731/ 3100 | consumed samples: 3545088 | consumed tokens: 7260340224 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.661413E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1732/ 3100 | consumed samples: 3547136 | consumed tokens: 7264534528 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.853907E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1733/ 3100 | consumed samples: 3549184 | consumed tokens: 7268728832 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.932019E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1734/ 3100 | consumed samples: 3551232 | consumed tokens: 7272923136 | elapsed time per iteration (s): 141.32 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.805575E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.492 | TFLOPs: 147.94 | -[default7]: iteration 1735/ 3100 | consumed samples: 3553280 | consumed tokens: 7277117440 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.671151E-01 | grad norm: 0.358 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 1736/ 3100 | consumed samples: 3555328 | consumed tokens: 7281311744 | elapsed time per iteration (s): 141.28 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.633033E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.496 | TFLOPs: 147.98 | -[default7]: iteration 1737/ 3100 | consumed samples: 3557376 | consumed tokens: 7285506048 | elapsed time per iteration (s): 141.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.677203E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.429 | TFLOPs: 147.30 | -[default7]: iteration 1738/ 3100 | consumed samples: 3559424 | consumed tokens: 7289700352 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.785817E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 1739/ 3100 | consumed samples: 3561472 | consumed tokens: 7293894656 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.777594E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 1740/ 3100 | consumed samples: 3563520 | consumed tokens: 7298088960 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.816712E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1741/ 3100 | consumed samples: 3565568 | consumed tokens: 7302283264 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.801876E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.26 | -[default7]: iteration 1742/ 3100 | consumed samples: 3567616 | consumed tokens: 7306477568 | elapsed time per iteration (s): 142.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.764611E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.420 | TFLOPs: 147.21 | -[default0]:saving checkpoint at iteration 1743 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-10 18:45:50,546] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1743 is begin to save! -[default7]: iteration 1743/ 3100 | consumed samples: 3569664 | consumed tokens: 7310671872 | elapsed time per iteration (s): 141.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.679150E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.428 | TFLOPs: 147.29 | -[default4]:[2022-09-10 18:45:50,577] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_63-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,577] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_62-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,683] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_71_model_states.pt... -[default0]:[2022-09-10 18:45:50,691] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_48-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,690] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_47-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,691] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_49-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,645] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_22-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,691] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_06-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,712] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_42-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,702] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_45-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,691] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_64-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,718] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_01-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,711] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_27-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_15-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,702] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_44-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,691] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_33-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,704] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_18-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,691] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_32-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,733] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_24-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,690] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_46-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,704] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_25-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,645] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_23-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,689] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_72-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,705] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_34-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,733] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_19-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,711] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_26-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,705] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_55-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,704] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_56-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,700] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_36-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,711] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_10-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,735] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_21-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,689] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_66-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,691] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_65-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,711] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_60-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,711] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_53-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,704] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_58-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,691] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_07-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,772] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_13-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,772] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_12-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,704] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_37-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,689] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_67-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,735] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_09-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,705] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_57-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_71-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,715] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_69-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_14-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,735] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_08-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_51-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,704] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_71_model_states.pt. -[default0]:[2022-09-10 18:45:50,721] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_04-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,711] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_29-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,721] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_05-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,764] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_68-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,735] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_28-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,705] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_54-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,704] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_50-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,711] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_30-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,772] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_40-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,718] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_03-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,735] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_31-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,800] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_39-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,735] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_41-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,735] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_20-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_35-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,774] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_11-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,757] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_17-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,770] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_52-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_70-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,800] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_38-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,766] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_59-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,773] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_43-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:50,775] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_16-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:50,796] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_61-model_00-model_states.pt... -[default4]:[2022-09-10 18:45:53,753] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_63-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:53,753] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_61_model_states.pt... -[default4]:[2022-09-10 18:45:53,760] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_61_model_states.pt. -[default0]:[2022-09-10 18:45:53,935] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_72-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:53,935] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_74-model_00-model_states.pt... -[default0]:[2022-09-10 18:45:53,938] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_74-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:53,938] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_70_model_states.pt... -[default0]:[2022-09-10 18:45:53,940] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_70_model_states.pt. -[default4]:[2022-09-10 18:45:54,153] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_49-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,154] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_47_model_states.pt... -[default4]:[2022-09-10 18:45:54,163] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_47_model_states.pt. -[default0]:[2022-09-10 18:45:54,188] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_40-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,188] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_38_model_states.pt... -[default0]:[2022-09-10 18:45:54,191] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_38_model_states.pt. -[default4]:[2022-09-10 18:45:54,179] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_35-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,179] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_33_model_states.pt... -[default4]:[2022-09-10 18:45:54,182] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_33_model_states.pt. -[default0]:[2022-09-10 18:45:54,232] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_62-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,232] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_60_model_states.pt... -[default0]:[2022-09-10 18:45:54,238] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_60_model_states.pt. -[default0]:[2022-09-10 18:45:54,240] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_56-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,240] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_54_model_states.pt... -[default0]:[2022-09-10 18:45:54,251] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_54_model_states.pt. -[default4]:[2022-09-10 18:45:54,240] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_07-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,240] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_05_model_states.pt... -[default4]:[2022-09-10 18:45:54,251] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_05_model_states.pt. -[default0]:[2022-09-10 18:45:54,210] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_14-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,211] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_12_model_states.pt... -[default0]:[2022-09-10 18:45:54,219] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_12_model_states.pt. -[default4]:[2022-09-10 18:45:54,238] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_47-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,238] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_45_model_states.pt... -[default4]:[2022-09-10 18:45:54,249] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_45_model_states.pt. -[default0]:[2022-09-10 18:45:54,278] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_22-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,279] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_20_model_states.pt... -[default0]:[2022-09-10 18:45:54,292] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_20_model_states.pt. -[default0]:[2022-09-10 18:45:54,295] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_34-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,296] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_32_model_states.pt... -[default0]:[2022-09-10 18:45:54,299] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_32_model_states.pt. -[default4]:[2022-09-10 18:45:54,278] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_43-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,278] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_41_model_states.pt... -[default4]:[2022-09-10 18:45:54,291] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_41_model_states.pt. -[default4]:[2022-09-10 18:45:54,349] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_61-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,349] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_59_model_states.pt... -[default4]:[2022-09-10 18:45:54,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_59_model_states.pt. -[default0]:[2022-09-10 18:45:54,363] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_48-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,363] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_46_model_states.pt... -[default0]:[2022-09-10 18:45:54,378] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_46_model_states.pt. -[default0]:[2022-09-10 18:45:54,333] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_06-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,333] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_04_model_states.pt... -[default0]:[2022-09-10 18:45:54,336] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_04_model_states.pt. -[default4]:[2022-09-10 18:45:54,365] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_03-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,365] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_01_model_states.pt... -[default4]:[2022-09-10 18:45:54,368] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_01_model_states.pt. -[default4]:[2022-09-10 18:45:54,366] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_31-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_29_model_states.pt... -[default4]:[2022-09-10 18:45:54,368] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_29_model_states.pt. -[default0]:[2022-09-10 18:45:54,386] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_46-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,387] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_44_model_states.pt... -[default0]:[2022-09-10 18:45:54,389] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_44_model_states.pt. -[default4]:[2022-09-10 18:45:54,406] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_23-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,406] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_21_model_states.pt... -[default4]:[2022-09-10 18:45:54,408] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_21_model_states.pt. -[default0]:[2022-09-10 18:45:54,438] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_66-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,438] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_64_model_states.pt... -[default0]:[2022-09-10 18:45:54,460] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_64_model_states.pt. -[default4]:[2022-09-10 18:45:54,463] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_57-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,463] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_55_model_states.pt... -[default4]:[2022-09-10 18:45:54,466] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_55_model_states.pt. -[default4]:[2022-09-10 18:45:54,421] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_05-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,421] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_03_model_states.pt... -[default4]:[2022-09-10 18:45:54,424] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_03_model_states.pt. -[default0]:[2022-09-10 18:45:54,486] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_42-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,487] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_40_model_states.pt... -[default0]:[2022-09-10 18:45:54,489] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_40_model_states.pt. -[default4]:[2022-09-10 18:45:54,504] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_41-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,505] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_39_model_states.pt... -[default4]:[2022-09-10 18:45:54,507] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_39_model_states.pt. -[default4]:[2022-09-10 18:45:54,473] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_15-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,473] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_13_model_states.pt... -[default4]:[2022-09-10 18:45:54,475] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_13_model_states.pt. -[default4]:[2022-09-10 18:45:54,531] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_33-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,532] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_31_model_states.pt... -[default4]:[2022-09-10 18:45:54,501] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_25-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,501] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_23_model_states.pt... -[default4]:[2022-09-10 18:45:54,504] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_23_model_states.pt. -[default0]:[2022-09-10 18:45:54,529] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_38-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,529] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_36_model_states.pt... -[default0]:[2022-09-10 18:45:54,534] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_36_model_states.pt. -[default4]:[2022-09-10 18:45:54,541] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_67-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,541] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_65_model_states.pt... -[default4]:[2022-09-10 18:45:54,545] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_65_model_states.pt. -[default0]:[2022-09-10 18:45:54,527] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_28-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,527] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_26_model_states.pt... -[default0]:[2022-09-10 18:45:54,532] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_26_model_states.pt. -[default0]:[2022-09-10 18:45:54,526] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_30-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,526] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_28_model_states.pt... -[default0]:[2022-09-10 18:45:54,529] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_28_model_states.pt. -[default0]:[2022-09-10 18:45:54,603] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_44-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,603] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_42_model_states.pt... -[default0]:[2022-09-10 18:45:54,608] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_42_model_states.pt. -[default4]:[2022-09-10 18:45:54,547] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_31_model_states.pt. -[default0]:[2022-09-10 18:45:54,575] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_32-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,575] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_30_model_states.pt... -[default0]:[2022-09-10 18:45:54,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_30_model_states.pt. -[default0]:[2022-09-10 18:45:54,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_24-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,547] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_22_model_states.pt... -[default0]:[2022-09-10 18:45:54,549] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_22_model_states.pt. -[default0]:[2022-09-10 18:45:54,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_36-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,637] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_34_model_states.pt... -[default0]:[2022-09-10 18:45:54,639] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_34_model_states.pt. -[default4]:[2022-09-10 18:45:54,623] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_21-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,623] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_19_model_states.pt... -[default4]:[2022-09-10 18:45:54,626] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_19_model_states.pt. -[default0]:[2022-09-10 18:45:54,617] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_60-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,617] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_58_model_states.pt... -[default0]:[2022-09-10 18:45:54,620] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_58_model_states.pt. -[default4]:[2022-09-10 18:45:54,657] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_13-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,658] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_11_model_states.pt... -[default4]:[2022-09-10 18:45:54,659] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_11_model_states.pt. -[default0]:[2022-09-10 18:45:54,601] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_12-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,602] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_10_model_states.pt... -[default0]:[2022-09-10 18:45:54,605] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_10_model_states.pt. -[default4]:[2022-09-10 18:45:54,630] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_71-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,630] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_69_model_states.pt... -[default4]:[2022-09-10 18:45:54,632] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_69_model_states.pt. -[default4]:[2022-09-10 18:45:54,621] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_69-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,622] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_67_model_states.pt... -[default4]:[2022-09-10 18:45:54,625] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_67_model_states.pt. -[default4]:[2022-09-10 18:45:54,688] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_51-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,688] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_49_model_states.pt... -[default4]:[2022-09-10 18:45:54,690] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_49_model_states.pt. -[default0]:[2022-09-10 18:45:54,655] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_04-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,656] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_02_model_states.pt... -[default0]:[2022-09-10 18:45:54,658] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_02_model_states.pt. -[default4]:[2022-09-10 18:45:54,645] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_29-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,645] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_27_model_states.pt... -[default4]:[2022-09-10 18:45:54,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_27_model_states.pt. -[default0]:[2022-09-10 18:45:54,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_68-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,636] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_66_model_states.pt... -[default0]:[2022-09-10 18:45:54,638] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_66_model_states.pt. -[default0]:[2022-09-10 18:45:54,635] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_50-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,635] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_48_model_states.pt... -[default0]:[2022-09-10 18:45:54,638] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_48_model_states.pt. -[default4]:[2022-09-10 18:45:54,651] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_39-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,652] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_37_model_states.pt... -[default4]:[2022-09-10 18:45:54,664] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_37_model_states.pt. -[default0]:[2022-09-10 18:45:54,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_20-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,675] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_18_model_states.pt... -[default0]:[2022-09-10 18:45:54,678] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_18_model_states.pt. -[default4]:[2022-09-10 18:45:54,637] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_45-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,637] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_43_model_states.pt... -[default4]:[2022-09-10 18:45:54,639] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_43_model_states.pt. -[default4]:[2022-09-10 18:45:54,713] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_27-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,713] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_25_model_states.pt... -[default4]:[2022-09-10 18:45:54,718] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_25_model_states.pt. -[default4]:[2022-09-10 18:45:54,716] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_11-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,717] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_09_model_states.pt... -[default4]:[2022-09-10 18:45:54,720] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_09_model_states.pt. -[default0]:[2022-09-10 18:45:54,661] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_52-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,661] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_50_model_states.pt... -[default0]:[2022-09-10 18:45:54,664] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_50_model_states.pt. -[default0]:[2022-09-10 18:45:54,659] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_70-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,660] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_68_model_states.pt... -[default0]:[2022-09-10 18:45:54,662] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_68_model_states.pt. -[default0]:[2022-09-10 18:45:54,732] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_26-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,733] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_24_model_states.pt... -[default0]:[2022-09-10 18:45:54,735] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_24_model_states.pt. -[default0]:[2022-09-10 18:45:54,676] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_16-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,676] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_14_model_states.pt... -[default0]:[2022-09-10 18:45:54,679] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_14_model_states.pt. -[default4]:[2022-09-10 18:45:54,762] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_53-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,762] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_51_model_states.pt... -[default4]:[2022-09-10 18:45:54,764] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_51_model_states.pt. -[default4]:[2022-09-10 18:45:54,730] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_37-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,730] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_35_model_states.pt... -[default4]:[2022-09-10 18:45:54,732] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_35_model_states.pt. -[default0]:[2022-09-10 18:45:54,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_54-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,782] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_52_model_states.pt... -[default0]:[2022-09-10 18:45:54,794] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_52_model_states.pt. -[default0]:[2022-09-10 18:45:54,834] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_18-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_16_model_states.pt... -[default0]:[2022-09-10 18:45:54,839] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_16_model_states.pt. -[default4]:[2022-09-10 18:45:54,747] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_17-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,748] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_15_model_states.pt... -[default4]:[2022-09-10 18:45:54,754] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_15_model_states.pt. -[default4]:[2022-09-10 18:45:54,767] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_59-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,768] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_57_model_states.pt... -[default4]:[2022-09-10 18:45:54,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_57_model_states.pt. -[default0]:[2022-09-10 18:45:54,772] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_10-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,772] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_08_model_states.pt... -[default0]:[2022-09-10 18:45:54,774] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_08_model_states.pt. -[default0]:[2022-09-10 18:45:54,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_58-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,832] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_56_model_states.pt... -[default0]:[2022-09-10 18:45:54,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_56_model_states.pt. -[default4]:[2022-09-10 18:45:54,858] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_19-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,859] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_17_model_states.pt... -[default4]:[2022-09-10 18:45:54,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_17_model_states.pt. -[default4]:[2022-09-10 18:45:54,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_55-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,854] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_53_model_states.pt... -[default4]:[2022-09-10 18:45:54,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_53_model_states.pt. -[default4]:[2022-09-10 18:45:54,889] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_09-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,889] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_07_model_states.pt... -[default4]:[2022-09-10 18:45:54,892] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_07_model_states.pt. -[default0]:[2022-09-10 18:45:54,897] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_08-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,898] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_06_model_states.pt... -[default0]:[2022-09-10 18:45:54,899] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_06_model_states.pt. -[default0]:[2022-09-10 18:45:54,988] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_64-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:54,989] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_62_model_states.pt... -[default0]:[2022-09-10 18:45:54,990] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_62_model_states.pt. -[default4]:[2022-09-10 18:45:54,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_65-model_00-model_states.pt. -[default4]:[2022-09-10 18:45:54,981] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_63_model_states.pt... -[default4]:[2022-09-10 18:45:54,985] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_63_model_states.pt. -[default0]:[2022-09-10 18:45:55,572] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/layer_01-model_00-model_states.pt. -[default0]:[2022-09-10 18:45:55,573] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_00_model_states.pt -[default0]:[2022-09-10 18:45:55,573] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_00_model_states.pt... -[default0]:[2022-09-10 18:45:55,575] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/mp_rank_00_model_states.pt. -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default3]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default5]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default7]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default0]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default6]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default1]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default4]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default2]:[2022-09-10 18:45:55,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default3]:[2022-09-10 18:46:04,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-10 18:46:04,222] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default1]:[2022-09-10 18:46:04,381] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-10 18:46:04,381] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default3]:[2022-09-10 18:46:04,585] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-10 18:46:04,586] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default6]:[2022-09-10 18:46:04,746] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-10 18:46:04,746] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default7]:[2022-09-10 18:46:04,814] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-10 18:46:04,814] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default5]:[2022-09-10 18:46:04,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-10 18:46:04,831] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default0]:[2022-09-10 18:46:04,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-10 18:46:04,865] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default4]:[2022-09-10 18:46:04,909] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-10 18:46:04,910] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default7]:[2022-09-10 18:46:04,950] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-10 18:46:04,950] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default1]:[2022-09-10 18:46:05,021] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-10 18:46:05,021] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default6]:[2022-09-10 18:46:05,116] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-10 18:46:05,116] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default7]:[2022-09-10 18:46:05,098] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-10 18:46:05,099] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default5]:[2022-09-10 18:46:05,157] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-10 18:46:05,157] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default6]:[2022-09-10 18:46:05,274] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-10 18:46:05,275] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default0]:[2022-09-10 18:46:05,291] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-10 18:46:05,291] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default4]:[2022-09-10 18:46:05,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-10 18:46:05,284] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default0]:[2022-09-10 18:46:05,364] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-10 18:46:05,364] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default0]:[2022-09-10 18:46:05,437] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-10 18:46:05,437] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default3]:[2022-09-10 18:46:05,490] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-10 18:46:05,490] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default3]:[2022-09-10 18:46:05,541] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-10 18:46:05,541] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default5]:[2022-09-10 18:46:05,544] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-10 18:46:05,544] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default4]:[2022-09-10 18:46:05,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-10 18:46:05,562] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default4]:[2022-09-10 18:46:05,609] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-10 18:46:05,610] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default3]:[2022-09-10 18:46:05,611] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-10 18:46:05,612] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default4]:[2022-09-10 18:46:05,705] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-10 18:46:05,706] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default2]:[2022-09-10 18:46:05,691] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-10 18:46:05,691] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default1]:[2022-09-10 18:46:05,683] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-10 18:46:05,683] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default0]:[2022-09-10 18:46:05,721] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-10 18:46:05,721] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default2]:[2022-09-10 18:46:05,704] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-10 18:46:05,704] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default1]:[2022-09-10 18:46:05,716] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-10 18:46:05,716] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default0]:[2022-09-10 18:46:05,731] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-10 18:46:05,731] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default5]:[2022-09-10 18:46:05,869] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-10 18:46:05,869] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default3]:[2022-09-10 18:46:05,893] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-10 18:46:05,893] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default6]:[2022-09-10 18:46:05,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-10 18:46:05,861] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default5]:[2022-09-10 18:46:05,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-10 18:46:05,850] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default7]:[2022-09-10 18:46:05,877] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-10 18:46:05,877] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default3]:[2022-09-10 18:46:05,956] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-10 18:46:05,957] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default6]:[2022-09-10 18:46:06,133] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-10 18:46:06,133] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default3]:[2022-09-10 18:46:06,160] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-10 18:46:06,160] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default3]:[2022-09-10 18:46:06,239] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-10 18:46:06,239] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default0]:[2022-09-10 18:46:06,231] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-10 18:46:06,231] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default1]:[2022-09-10 18:46:06,334] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-10 18:46:06,334] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default7]:[2022-09-10 18:46:06,376] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-10 18:46:06,376] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default7]:[2022-09-10 18:46:06,428] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-10 18:46:06,428] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default0]:[2022-09-10 18:46:06,381] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-10 18:46:06,382] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default2]:[2022-09-10 18:46:06,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-10 18:46:06,456] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default4]:[2022-09-10 18:46:06,523] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-10 18:46:06,523] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default0]:[2022-09-10 18:46:06,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-10 18:46:06,502] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default2]:[2022-09-10 18:46:06,542] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-10 18:46:06,542] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default4]:[2022-09-10 18:46:06,548] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-10 18:46:06,548] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default5]:[2022-09-10 18:46:06,483] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-10 18:46:06,483] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default7]:[2022-09-10 18:46:06,536] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-10 18:46:06,536] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default1]:[2022-09-10 18:46:06,571] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-10 18:46:06,571] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default3]:[2022-09-10 18:46:06,587] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-10 18:46:06,587] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default2]:[2022-09-10 18:46:06,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-10 18:46:06,647] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default0]:[2022-09-10 18:46:06,650] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-10 18:46:06,651] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default5]:[2022-09-10 18:46:06,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-10 18:46:06,577] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default5]:[2022-09-10 18:46:06,657] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-10 18:46:06,657] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default4]:[2022-09-10 18:46:06,611] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-10 18:46:06,611] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default0]:[2022-09-10 18:46:06,623] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-10 18:46:06,623] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default5]:[2022-09-10 18:46:06,699] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-10 18:46:06,699] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default1]:[2022-09-10 18:46:06,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-10 18:46:06,675] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default6]:[2022-09-10 18:46:06,726] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-10 18:46:06,727] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default7]:[2022-09-10 18:46:06,745] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-10 18:46:06,745] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default2]:[2022-09-10 18:46:06,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-10 18:46:06,800] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default6]:[2022-09-10 18:46:06,766] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-10 18:46:06,766] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default4]:[2022-09-10 18:46:06,762] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-10 18:46:06,762] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default2]:[2022-09-10 18:46:06,760] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-10 18:46:06,760] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default3]:[2022-09-10 18:46:06,863] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-10 18:46:06,863] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default2]:[2022-09-10 18:46:06,828] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-10 18:46:06,828] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default6]:[2022-09-10 18:46:06,843] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-10 18:46:06,843] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default1]:[2022-09-10 18:46:06,876] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-10 18:46:06,876] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default3]:[2022-09-10 18:46:06,859] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-10 18:46:06,859] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default4]:[2022-09-10 18:46:06,886] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-10 18:46:06,886] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default2]:[2022-09-10 18:46:06,912] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-10 18:46:06,912] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default6]:[2022-09-10 18:46:06,942] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-10 18:46:06,942] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default7]:[2022-09-10 18:46:06,983] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-10 18:46:06,983] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default0]:[2022-09-10 18:46:06,974] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-10 18:46:06,974] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default0]:[2022-09-10 18:46:06,955] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-10 18:46:06,955] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default2]:[2022-09-10 18:46:07,055] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-10 18:46:07,055] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default6]:[2022-09-10 18:46:07,026] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-10 18:46:07,026] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default4]:[2022-09-10 18:46:07,014] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-10 18:46:07,014] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default1]:[2022-09-10 18:46:07,048] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-10 18:46:07,048] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default1]:[2022-09-10 18:46:07,104] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-10 18:46:07,104] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default6]:[2022-09-10 18:46:07,080] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-10 18:46:07,080] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default0]:[2022-09-10 18:46:07,155] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-10 18:46:07,155] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default6]:[2022-09-10 18:46:07,083] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-10 18:46:07,083] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default3]:[2022-09-10 18:46:07,173] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-10 18:46:07,174] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default4]:[2022-09-10 18:46:07,146] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-10 18:46:07,146] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default3]:[2022-09-10 18:46:07,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-10 18:46:07,097] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default2]:[2022-09-10 18:46:07,174] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-10 18:46:07,174] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default4]:[2022-09-10 18:46:07,232] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-10 18:46:07,233] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default0]:[2022-09-10 18:46:07,212] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-10 18:46:07,212] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default5]:[2022-09-10 18:46:07,249] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-10 18:46:07,249] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default0]:[2022-09-10 18:46:07,234] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-10 18:46:07,234] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default5]:[2022-09-10 18:46:07,259] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-10 18:46:07,259] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default1]:[2022-09-10 18:46:07,288] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-10 18:46:07,288] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default6]:[2022-09-10 18:46:07,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-10 18:46:07,276] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default0]:[2022-09-10 18:46:07,277] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-10 18:46:07,278] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default7]:[2022-09-10 18:46:07,300] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-10 18:46:07,300] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default0]:[2022-09-10 18:46:07,286] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-10 18:46:07,286] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default5]:[2022-09-10 18:46:07,300] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-10 18:46:07,300] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default7]:[2022-09-10 18:46:07,346] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-10 18:46:07,346] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default4]:[2022-09-10 18:46:07,340] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-10 18:46:07,341] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default2]:[2022-09-10 18:46:07,366] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-10 18:46:07,366] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default3]:[2022-09-10 18:46:07,404] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-10 18:46:07,404] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default7]:[2022-09-10 18:46:07,325] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-10 18:46:07,325] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default3]:[2022-09-10 18:46:07,433] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-10 18:46:07,434] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default0]:[2022-09-10 18:46:07,412] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-10 18:46:07,412] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default5]:[2022-09-10 18:46:07,410] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-10 18:46:07,410] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default2]:[2022-09-10 18:46:07,452] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-10 18:46:07,452] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default3]:[2022-09-10 18:46:07,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-10 18:46:07,416] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default2]:[2022-09-10 18:46:07,481] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-10 18:46:07,481] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default2]:[2022-09-10 18:46:07,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-10 18:46:07,503] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default3]:[2022-09-10 18:46:07,515] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-10 18:46:07,515] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default1]:[2022-09-10 18:46:07,433] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-10 18:46:07,433] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default3]:[2022-09-10 18:46:07,457] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-10 18:46:07,457] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default3]:[2022-09-10 18:46:07,438] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-10 18:46:07,438] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default5]:[2022-09-10 18:46:07,459] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-10 18:46:07,459] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default0]:[2022-09-10 18:46:07,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-10 18:46:07,455] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default5]:[2022-09-10 18:46:07,587] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-10 18:46:07,587] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default6]:[2022-09-10 18:46:07,521] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-10 18:46:07,521] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default7]:[2022-09-10 18:46:07,536] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-10 18:46:07,536] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default3]:[2022-09-10 18:46:07,532] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-10 18:46:07,532] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default5]:[2022-09-10 18:46:07,523] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-10 18:46:07,523] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default2]:[2022-09-10 18:46:07,591] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-10 18:46:07,591] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default7]:[2022-09-10 18:46:07,545] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-10 18:46:07,546] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default7]:[2022-09-10 18:46:07,625] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-10 18:46:07,625] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default2]:[2022-09-10 18:46:07,593] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-10 18:46:07,593] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default5]:[2022-09-10 18:46:07,609] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-10 18:46:07,609] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default5]:[2022-09-10 18:46:07,588] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-10 18:46:07,588] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default4]:[2022-09-10 18:46:07,646] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-10 18:46:07,646] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default6]:[2022-09-10 18:46:07,704] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-10 18:46:07,704] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default0]:[2022-09-10 18:46:07,635] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-10 18:46:07,635] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default6]:[2022-09-10 18:46:07,740] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-10 18:46:07,741] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default4]:[2022-09-10 18:46:07,738] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-10 18:46:07,738] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default7]:[2022-09-10 18:46:07,748] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-10 18:46:07,748] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default2]:[2022-09-10 18:46:07,705] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-10 18:46:07,705] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default1]:[2022-09-10 18:46:07,726] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-10 18:46:07,726] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default0]:[2022-09-10 18:46:07,741] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-10 18:46:07,741] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default2]:[2022-09-10 18:46:07,733] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-10 18:46:07,733] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default3]:[2022-09-10 18:46:07,793] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-10 18:46:07,793] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default7]:[2022-09-10 18:46:07,818] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-10 18:46:07,818] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default5]:[2022-09-10 18:46:07,877] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-10 18:46:07,877] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default2]:[2022-09-10 18:46:07,892] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-10 18:46:07,892] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default4]:[2022-09-10 18:46:07,889] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-10 18:46:07,889] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default6]:[2022-09-10 18:46:07,920] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-10 18:46:07,920] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default6]:[2022-09-10 18:46:07,881] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-10 18:46:07,882] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default3]:[2022-09-10 18:46:07,974] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-10 18:46:07,974] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default1]:[2022-09-10 18:46:07,966] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-10 18:46:07,966] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default4]:[2022-09-10 18:46:07,989] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-10 18:46:07,989] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default6]:[2022-09-10 18:46:07,946] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-10 18:46:07,946] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default0]:[2022-09-10 18:46:07,991] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-10 18:46:07,991] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default4]:[2022-09-10 18:46:07,988] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-10 18:46:07,988] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default6]:[2022-09-10 18:46:08,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-10 18:46:08,036] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default0]:[2022-09-10 18:46:08,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-10 18:46:08,016] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default2]:[2022-09-10 18:46:08,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-10 18:46:08,004] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default4]:[2022-09-10 18:46:08,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-10 18:46:08,077] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default7]:[2022-09-10 18:46:08,090] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-10 18:46:08,090] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default4]:[2022-09-10 18:46:08,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-10 18:46:08,079] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default1]:[2022-09-10 18:46:08,157] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-10 18:46:08,157] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default0]:[2022-09-10 18:46:08,146] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-10 18:46:08,146] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default1]:[2022-09-10 18:46:08,155] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-10 18:46:08,155] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default4]:[2022-09-10 18:46:08,245] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-10 18:46:08,245] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default0]:[2022-09-10 18:46:08,206] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-10 18:46:08,207] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default5]:[2022-09-10 18:46:08,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-10 18:46:08,276] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default5]:[2022-09-10 18:46:08,254] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-10 18:46:08,254] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default4]:[2022-09-10 18:46:08,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-10 18:46:08,310] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default6]:[2022-09-10 18:46:08,279] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-10 18:46:08,279] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default7]:[2022-09-10 18:46:08,296] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-10 18:46:08,296] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default1]:[2022-09-10 18:46:08,302] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-10 18:46:08,302] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default2]:[2022-09-10 18:46:08,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-10 18:46:08,313] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default4]:[2022-09-10 18:46:08,328] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-10 18:46:08,328] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default4]:[2022-09-10 18:46:08,394] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-10 18:46:08,394] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default6]:[2022-09-10 18:46:08,373] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-10 18:46:08,373] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default7]:[2022-09-10 18:46:08,392] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-10 18:46:08,392] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default5]:[2022-09-10 18:46:08,432] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-10 18:46:08,433] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default1]:[2022-09-10 18:46:08,349] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-10 18:46:08,350] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default6]:[2022-09-10 18:46:08,409] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-10 18:46:08,409] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default7]:[2022-09-10 18:46:08,383] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-10 18:46:08,383] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default5]:[2022-09-10 18:46:08,394] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-10 18:46:08,394] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default2]:[2022-09-10 18:46:08,409] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-10 18:46:08,409] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default1]:[2022-09-10 18:46:08,415] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-10 18:46:08,415] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default1]:[2022-09-10 18:46:08,457] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-10 18:46:08,457] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default7]:[2022-09-10 18:46:08,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-10 18:46:08,455] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default5]:[2022-09-10 18:46:08,547] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-10 18:46:08,547] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default6]:[2022-09-10 18:46:08,514] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-10 18:46:08,515] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default5]:[2022-09-10 18:46:08,496] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-10 18:46:08,496] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default3]:[2022-09-10 18:46:08,495] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-10 18:46:08,495] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default7]:[2022-09-10 18:46:08,542] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-10 18:46:08,543] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default0]:[2022-09-10 18:46:08,493] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-10 18:46:08,493] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default7]:[2022-09-10 18:46:08,590] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-10 18:46:08,591] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default2]:[2022-09-10 18:46:08,570] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-10 18:46:08,570] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default1]:[2022-09-10 18:46:08,554] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-10 18:46:08,554] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default7]:[2022-09-10 18:46:08,599] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-10 18:46:08,599] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default6]:[2022-09-10 18:46:08,623] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-10 18:46:08,623] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default6]:[2022-09-10 18:46:08,642] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-10 18:46:08,642] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default4]:[2022-09-10 18:46:08,679] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-10 18:46:08,680] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default7]:[2022-09-10 18:46:08,723] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-10 18:46:08,723] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default2]:[2022-09-10 18:46:08,745] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-10 18:46:08,745] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default1]:[2022-09-10 18:46:08,709] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-10 18:46:08,710] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default1]:[2022-09-10 18:46:08,705] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-10 18:46:08,705] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default1]:[2022-09-10 18:46:08,721] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-10 18:46:08,721] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default5]:[2022-09-10 18:46:08,712] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-10 18:46:08,712] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default2]:[2022-09-10 18:46:08,799] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-10 18:46:08,799] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default3]:[2022-09-10 18:46:08,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-10 18:46:08,770] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default5]:[2022-09-10 18:46:08,752] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-10 18:46:08,752] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default6]:[2022-09-10 18:46:08,826] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-10 18:46:08,826] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default3]:[2022-09-10 18:46:08,834] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-10 18:46:08,834] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default7]:[2022-09-10 18:46:08,929] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-10 18:46:08,930] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default1]:[2022-09-10 18:46:08,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-10 18:46:08,932] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default3]:[2022-09-10 18:46:08,931] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-10 18:46:08,931] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default4]:[2022-09-10 18:46:08,905] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-10 18:46:08,905] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default1]:[2022-09-10 18:46:09,099] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-10 18:46:09,099] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default1]:[2022-09-10 18:46:09,104] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-10 18:46:09,104] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default3]:[2022-09-10 18:46:09,154] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-10 18:46:09,154] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default2]:[2022-09-10 18:46:09,280] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-10 18:46:09,280] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default4]:[2022-09-10 18:46:09,363] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-10 18:46:09,363] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default5]:[2022-09-10 18:46:09,323] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-10 18:46:09,323] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default0]:[2022-09-10 18:46:09,387] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-10 18:46:09,387] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default2]:[2022-09-10 18:46:09,402] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-10 18:46:09,402] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default5]:[2022-09-10 18:46:09,314] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-10 18:46:09,314] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default4]:[2022-09-10 18:46:09,349] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-10 18:46:09,349] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default7]:[2022-09-10 18:46:09,405] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-10 18:46:09,405] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default3]:[2022-09-10 18:46:09,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-10 18:46:09,455] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default7]:[2022-09-10 18:46:09,518] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-10 18:46:09,518] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default1]:[2022-09-10 18:46:09,644] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-10 18:46:09,644] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default6]:[2022-09-10 18:46:09,668] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-10 18:46:09,668] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default6]:[2022-09-10 18:46:09,984] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-10 18:46:09,984] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default2]:[2022-09-10 18:46:10,753] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-10 18:46:10,754] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default3]:[2022-09-10 18:46:11,360] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-10 18:46:11,360] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default5]:[2022-09-10 18:46:11,338] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-10 18:46:11,338] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default2]:[2022-09-10 18:46:11,376] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-10 18:46:11,376] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default1]:[2022-09-10 18:46:11,736] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-10 18:46:11,736] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default0]:[2022-09-10 18:46:11,840] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-10 18:46:11,840] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default7]:[2022-09-10 18:46:12,139] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-10 18:46:12,139] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default6]:[2022-09-10 18:46:12,184] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-10 18:46:12,184] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default1]:[2022-09-10 18:46:12,274] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-10 18:46:12,274] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default0]:[2022-09-10 18:46:12,405] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-10 18:46:12,405] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default4]:[2022-09-10 18:46:13,058] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-10 18:46:13,058] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default4]:[2022-09-10 18:46:14,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-10 18:46:14,052] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default7]:[2022-09-10 18:46:14,240] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-10 18:46:14,240] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default5]:[2022-09-10 18:46:14,472] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-10 18:46:14,472] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default2]:[2022-09-10 18:46:14,622] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-10 18:46:14,623] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default3]:[2022-09-10 18:46:14,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-10 18:46:14,797] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default5]:[2022-09-10 18:46:14,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-10 18:46:14,837] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default4]:[2022-09-10 18:46:14,920] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-10 18:46:14,920] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default0]:[2022-09-10 18:46:15,153] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-10 18:46:15,153] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default1]:[2022-09-10 18:46:15,332] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-10 18:46:15,332] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default6]:[2022-09-10 18:46:15,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-10 18:46:15,408] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default7]:[2022-09-10 18:46:15,482] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-10 18:46:15,482] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default3]:[2022-09-10 18:46:15,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-10 18:46:15,561] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default2]:[2022-09-10 18:46:15,551] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-10 18:46:15,551] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default1]:[2022-09-10 18:46:15,612] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-10 18:46:15,612] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default5]:[2022-09-10 18:46:15,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-10 18:46:15,728] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default0]:[2022-09-10 18:46:15,780] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-10 18:46:15,781] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default6]:[2022-09-10 18:46:15,857] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-10 18:46:15,857] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default2]:[2022-09-10 18:46:15,966] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-10 18:46:15,966] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default0]:[2022-09-10 18:46:15,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-10 18:46:15,934] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default3]:[2022-09-10 18:46:16,019] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-10 18:46:16,019] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default3]:[2022-09-10 18:46:16,564] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-10 18:46:16,564] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default3]:[2022-09-10 18:46:17,624] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-10 18:46:17,624] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default1]:[2022-09-10 18:46:17,715] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-10 18:46:17,715] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default2]:[2022-09-10 18:46:18,328] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-10 18:46:18,329] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default3]:[2022-09-10 18:46:18,525] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-10 18:46:18,525] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default0]:[2022-09-10 18:46:18,720] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-10 18:46:18,720] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default1]:[2022-09-10 18:46:18,989] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-10 18:46:18,989] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default4]:[2022-09-10 18:46:19,107] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-10 18:46:19,107] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default2]:[2022-09-10 18:46:20,299] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-10 18:46:20,299] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default0]:[2022-09-10 18:46:20,331] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-10 18:46:20,331] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default1]:[2022-09-10 18:46:20,439] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-10 18:46:20,439] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default2]:[2022-09-10 18:46:20,637] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-10 18:46:20,637] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default4]:[2022-09-10 18:46:21,883] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-10 18:46:21,883] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default5]:[2022-09-10 18:46:23,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-10 18:46:23,672] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default7]:[2022-09-10 18:46:23,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-10 18:46:23,771] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default6]:[2022-09-10 18:46:24,098] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-10 18:46:24,098] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default7]:[2022-09-10 18:46:25,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-10 18:46:25,046] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default6]:[2022-09-10 18:46:25,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-10 18:46:25,157] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default6]:[2022-09-10 18:46:25,632] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-10 18:46:25,632] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default7]:[2022-09-10 18:46:25,659] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-10 18:46:25,660] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default1]:[2022-09-10 18:46:26,480] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-10 18:46:26,480] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default4]:[2022-09-10 18:46:26,558] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-10 18:46:26,558] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default5]:[2022-09-10 18:46:26,506] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-10 18:46:26,506] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default0]:[2022-09-10 18:46:26,571] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-10 18:46:26,573] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default6]:[2022-09-10 18:46:26,684] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-10 18:46:26,685] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default7]:[2022-09-10 18:46:26,688] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-10 18:46:26,688] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default4]:[2022-09-10 18:46:27,548] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-10 18:46:27,548] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]: successfully saved checkpoint at iteration 1743 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:time (ms) | save-checkpoint: 37107.27 -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,650] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-10 18:46:27,650] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1743/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default4]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default5]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default1]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default0]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default2]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default6]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default3]:[2022-09-10 18:46:27,651] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1743 is ready now! -[default7]: iteration 1744/ 3100 | consumed samples: 3571712 | consumed tokens: 7314866176 | elapsed time per iteration (s): 177.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.629517E-01 | grad norm: 0.585 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.532 | TFLOPs: 117.72 | -[default7]: iteration 1745/ 3100 | consumed samples: 3573760 | consumed tokens: 7319060480 | elapsed time per iteration (s): 141.30 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.690904E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.494 | TFLOPs: 147.96 | -[default7]: iteration 1746/ 3100 | consumed samples: 3575808 | consumed tokens: 7323254784 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.776172E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.26 | -[default7]: iteration 1747/ 3100 | consumed samples: 3577856 | consumed tokens: 7327449088 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.613179E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 1748/ 3100 | consumed samples: 3579904 | consumed tokens: 7331643392 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.553729E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1749/ 3100 | consumed samples: 3581952 | consumed tokens: 7335837696 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.584809E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.43 | -[default7]: iteration 1750/ 3100 | consumed samples: 3584000 | consumed tokens: 7340032000 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.627881E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]:validation_pretraining loss at iteration 1750 | lm loss value: 2.415612E+00 | lm loss PPL: 1.119662E+01 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]: iteration 1751/ 3100 | consumed samples: 3586048 | consumed tokens: 7344226304 | elapsed time per iteration (s): 182.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.660196E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.209 | TFLOPs: 114.42 | -[default7]: iteration 1752/ 3100 | consumed samples: 3588096 | consumed tokens: 7348420608 | elapsed time per iteration (s): 140.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.625915E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.537 | TFLOPs: 148.40 | -[default7]: iteration 1753/ 3100 | consumed samples: 3590144 | consumed tokens: 7352614912 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.603418E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 1754/ 3100 | consumed samples: 3592192 | consumed tokens: 7356809216 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.831358E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1755/ 3100 | consumed samples: 3594240 | consumed tokens: 7361003520 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.701701E-01 | grad norm: 0.405 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 1756/ 3100 | consumed samples: 3596288 | consumed tokens: 7365197824 | elapsed time per iteration (s): 141.16 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.772937E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.508 | TFLOPs: 148.10 | -[default7]: iteration 1757/ 3100 | consumed samples: 3598336 | consumed tokens: 7369392128 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.749734E-01 | grad norm: 0.420 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 1758/ 3100 | consumed samples: 3600384 | consumed tokens: 7373586432 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.795326E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 1759/ 3100 | consumed samples: 3602432 | consumed tokens: 7377780736 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.683563E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.45 | -[default7]: iteration 1760/ 3100 | consumed samples: 3604480 | consumed tokens: 7381975040 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.715372E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 1761/ 3100 | consumed samples: 3606528 | consumed tokens: 7386169344 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.761811E-01 | grad norm: 0.350 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 1762/ 3100 | consumed samples: 3608576 | consumed tokens: 7390363648 | elapsed time per iteration (s): 140.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.656982E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.550 | TFLOPs: 148.53 | -[default7]: iteration 1763/ 3100 | consumed samples: 3610624 | consumed tokens: 7394557952 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.718623E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.32 | -[default7]: iteration 1764/ 3100 | consumed samples: 3612672 | consumed tokens: 7398752256 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.703084E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.436 | TFLOPs: 147.37 | -[default7]: iteration 1765/ 3100 | consumed samples: 3614720 | consumed tokens: 7402946560 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.722554E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.34 | -[default7]: iteration 1766/ 3100 | consumed samples: 3616768 | consumed tokens: 7407140864 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.699067E-01 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1767/ 3100 | consumed samples: 3618816 | consumed tokens: 7411335168 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.706321E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.499 | TFLOPs: 148.01 | -[default7]: iteration 1768/ 3100 | consumed samples: 3620864 | consumed tokens: 7415529472 | elapsed time per iteration (s): 140.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.743511E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.553 | TFLOPs: 148.56 | -[default7]: iteration 1769/ 3100 | consumed samples: 3622912 | consumed tokens: 7419723776 | elapsed time per iteration (s): 140.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.685377E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.567 | TFLOPs: 148.70 | -[default7]: iteration 1770/ 3100 | consumed samples: 3624960 | consumed tokens: 7423918080 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.706649E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1771/ 3100 | consumed samples: 3627008 | consumed tokens: 7428112384 | elapsed time per iteration (s): 141.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.766316E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.427 | TFLOPs: 147.28 | -[default7]: iteration 1772/ 3100 | consumed samples: 3629056 | consumed tokens: 7432306688 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.684712E-01 | grad norm: 0.358 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1773/ 3100 | consumed samples: 3631104 | consumed tokens: 7436500992 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.746506E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1774/ 3100 | consumed samples: 3633152 | consumed tokens: 7440695296 | elapsed time per iteration (s): 141.96 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.526663E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.27 | -[default7]: iteration 1775/ 3100 | consumed samples: 3635200 | consumed tokens: 7444889600 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.799975E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.26 | -[default7]: iteration 1776/ 3100 | consumed samples: 3637248 | consumed tokens: 7449083904 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.870808E-01 | grad norm: 0.382 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.33 | -[default7]: iteration 1777/ 3100 | consumed samples: 3639296 | consumed tokens: 7453278208 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.707506E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.35 | -[default7]: iteration 1778/ 3100 | consumed samples: 3641344 | consumed tokens: 7457472512 | elapsed time per iteration (s): 140.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.711693E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.581 | TFLOPs: 148.85 | -[default7]: iteration 1779/ 3100 | consumed samples: 3643392 | consumed tokens: 7461666816 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.684831E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.433 | TFLOPs: 147.34 | -[default7]: iteration 1780/ 3100 | consumed samples: 3645440 | consumed tokens: 7465861120 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.745515E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1781/ 3100 | consumed samples: 3647488 | consumed tokens: 7470055424 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.611104E-01 | grad norm: 0.370 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1782/ 3100 | consumed samples: 3649536 | consumed tokens: 7474249728 | elapsed time per iteration (s): 141.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.718282E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.429 | TFLOPs: 147.30 | -[default7]: iteration 1783/ 3100 | consumed samples: 3651584 | consumed tokens: 7478444032 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.565354E-01 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1784/ 3100 | consumed samples: 3653632 | consumed tokens: 7482638336 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.732988E-01 | grad norm: 0.434 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 1785/ 3100 | consumed samples: 3655680 | consumed tokens: 7486832640 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.612861E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.581 | TFLOPs: 148.85 | -[default7]: iteration 1786/ 3100 | consumed samples: 3657728 | consumed tokens: 7491026944 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.626944E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 1787/ 3100 | consumed samples: 3659776 | consumed tokens: 7495221248 | elapsed time per iteration (s): 142.04 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.716832E-01 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.419 | TFLOPs: 147.20 | -[default7]: iteration 1788/ 3100 | consumed samples: 3661824 | consumed tokens: 7499415552 | elapsed time per iteration (s): 139.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.689469E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.673 | TFLOPs: 149.79 | -[default7]: iteration 1789/ 3100 | consumed samples: 3663872 | consumed tokens: 7503609856 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.742333E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1790/ 3100 | consumed samples: 3665920 | consumed tokens: 7507804160 | elapsed time per iteration (s): 140.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.738832E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.569 | TFLOPs: 148.73 | -[default7]: iteration 1791/ 3100 | consumed samples: 3667968 | consumed tokens: 7511998464 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.691047E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1792/ 3100 | consumed samples: 3670016 | consumed tokens: 7516192768 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.766300E-01 | grad norm: 0.432 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 1793/ 3100 | consumed samples: 3672064 | consumed tokens: 7520387072 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.821548E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 1794/ 3100 | consumed samples: 3674112 | consumed tokens: 7524581376 | elapsed time per iteration (s): 140.29 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.731304E-01 | grad norm: 2.267 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.598 | TFLOPs: 149.03 | -[default7]: iteration 1795/ 3100 | consumed samples: 3676160 | consumed tokens: 7528775680 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 8.536211E-01 | grad norm: 7.194 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 1796/ 3100 | consumed samples: 3678208 | consumed tokens: 7532969984 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.682809E-01 | grad norm: 1.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1797/ 3100 | consumed samples: 3680256 | consumed tokens: 7537164288 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.766413E-01 | grad norm: 1.352 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.78 | -[default7]: iteration 1798/ 3100 | consumed samples: 3682304 | consumed tokens: 7541358592 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.638492E-01 | grad norm: 0.828 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.62 | -[default7]: iteration 1799/ 3100 | consumed samples: 3684352 | consumed tokens: 7545552896 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.764723E-01 | grad norm: 0.443 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1800/ 3100 | consumed samples: 3686400 | consumed tokens: 7549747200 | elapsed time per iteration (s): 139.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.703511E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.632 | TFLOPs: 149.37 | -[default7]: iteration 1801/ 3100 | consumed samples: 3688448 | consumed tokens: 7553941504 | elapsed time per iteration (s): 140.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.731940E-01 | grad norm: 0.737 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.566 | TFLOPs: 148.69 | -[default7]: iteration 1802/ 3100 | consumed samples: 3690496 | consumed tokens: 7558135808 | elapsed time per iteration (s): 142.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.889036E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.423 | TFLOPs: 147.23 | -[default7]: iteration 1803/ 3100 | consumed samples: 3692544 | consumed tokens: 7562330112 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.614892E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1804/ 3100 | consumed samples: 3694592 | consumed tokens: 7566524416 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.856529E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 1805/ 3100 | consumed samples: 3696640 | consumed tokens: 7570718720 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.636346E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 1806/ 3100 | consumed samples: 3698688 | consumed tokens: 7574913024 | elapsed time per iteration (s): 140.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.507955E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.566 | TFLOPs: 148.70 | -[default7]: iteration 1807/ 3100 | consumed samples: 3700736 | consumed tokens: 7579107328 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.733989E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 1808/ 3100 | consumed samples: 3702784 | consumed tokens: 7583301632 | elapsed time per iteration (s): 141.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.674406E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.488 | TFLOPs: 147.90 | -[default7]: iteration 1809/ 3100 | consumed samples: 3704832 | consumed tokens: 7587495936 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.662557E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1810/ 3100 | consumed samples: 3706880 | consumed tokens: 7591690240 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.629541E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 1811/ 3100 | consumed samples: 3708928 | consumed tokens: 7595884544 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.646798E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 1812/ 3100 | consumed samples: 3710976 | consumed tokens: 7600078848 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.750329E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.442 | TFLOPs: 147.43 | -[default7]: iteration 1813/ 3100 | consumed samples: 3713024 | consumed tokens: 7604273152 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.696699E-01 | grad norm: 0.980 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 1814/ 3100 | consumed samples: 3715072 | consumed tokens: 7608467456 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.686957E-01 | grad norm: 0.416 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 1815/ 3100 | consumed samples: 3717120 | consumed tokens: 7612661760 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.575092E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 1816/ 3100 | consumed samples: 3719168 | consumed tokens: 7616856064 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.684633E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 1817/ 3100 | consumed samples: 3721216 | consumed tokens: 7621050368 | elapsed time per iteration (s): 140.98 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.824525E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.527 | TFLOPs: 148.30 | -[default7]: iteration 1818/ 3100 | consumed samples: 3723264 | consumed tokens: 7625244672 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.608542E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 1819/ 3100 | consumed samples: 3725312 | consumed tokens: 7629438976 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.610368E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 1820/ 3100 | consumed samples: 3727360 | consumed tokens: 7633633280 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.479916E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 1821/ 3100 | consumed samples: 3729408 | consumed tokens: 7637827584 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.592297E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1822/ 3100 | consumed samples: 3731456 | consumed tokens: 7642021888 | elapsed time per iteration (s): 141.38 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.723477E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.486 | TFLOPs: 147.88 | -[default7]: iteration 1823/ 3100 | consumed samples: 3733504 | consumed tokens: 7646216192 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.587206E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 1824/ 3100 | consumed samples: 3735552 | consumed tokens: 7650410496 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.752563E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1825/ 3100 | consumed samples: 3737600 | consumed tokens: 7654604800 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.713789E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1826/ 3100 | consumed samples: 3739648 | consumed tokens: 7658799104 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.854016E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 1827/ 3100 | consumed samples: 3741696 | consumed tokens: 7662993408 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.684876E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 1828/ 3100 | consumed samples: 3743744 | consumed tokens: 7667187712 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.673081E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 1829/ 3100 | consumed samples: 3745792 | consumed tokens: 7671382016 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.866893E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1830/ 3100 | consumed samples: 3747840 | consumed tokens: 7675576320 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.627202E-01 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1831/ 3100 | consumed samples: 3749888 | consumed tokens: 7679770624 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.640256E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 1832/ 3100 | consumed samples: 3751936 | consumed tokens: 7683964928 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.802217E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 1833/ 3100 | consumed samples: 3753984 | consumed tokens: 7688159232 | elapsed time per iteration (s): 141.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.403827E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.484 | TFLOPs: 147.86 | -[default7]: iteration 1834/ 3100 | consumed samples: 3756032 | consumed tokens: 7692353536 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.672205E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 1835/ 3100 | consumed samples: 3758080 | consumed tokens: 7696547840 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.565379E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.430 | TFLOPs: 147.31 | -[default7]: iteration 1836/ 3100 | consumed samples: 3760128 | consumed tokens: 7700742144 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.540829E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 1837/ 3100 | consumed samples: 3762176 | consumed tokens: 7704936448 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.709472E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1838/ 3100 | consumed samples: 3764224 | consumed tokens: 7709130752 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.488993E-01 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1839/ 3100 | consumed samples: 3766272 | consumed tokens: 7713325056 | elapsed time per iteration (s): 141.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.592387E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 1840/ 3100 | consumed samples: 3768320 | consumed tokens: 7717519360 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.632158E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1841/ 3100 | consumed samples: 3770368 | consumed tokens: 7721713664 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.584251E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1842/ 3100 | consumed samples: 3772416 | consumed tokens: 7725907968 | elapsed time per iteration (s): 142.03 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.642658E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.420 | TFLOPs: 147.20 | -[default7]: iteration 1843/ 3100 | consumed samples: 3774464 | consumed tokens: 7730102272 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.610057E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1844/ 3100 | consumed samples: 3776512 | consumed tokens: 7734296576 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.581670E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.430 | TFLOPs: 147.31 | -[default7]: iteration 1845/ 3100 | consumed samples: 3778560 | consumed tokens: 7738490880 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.504508E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 1846/ 3100 | consumed samples: 3780608 | consumed tokens: 7742685184 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.565956E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1847/ 3100 | consumed samples: 3782656 | consumed tokens: 7746879488 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.648281E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1848/ 3100 | consumed samples: 3784704 | consumed tokens: 7751073792 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.476110E-01 | grad norm: 0.339 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 1849/ 3100 | consumed samples: 3786752 | consumed tokens: 7755268096 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.686744E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 1850/ 3100 | consumed samples: 3788800 | consumed tokens: 7759462400 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.611716E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1851/ 3100 | consumed samples: 3790848 | consumed tokens: 7763656704 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.570992E-01 | grad norm: 0.353 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 1852/ 3100 | consumed samples: 3792896 | consumed tokens: 7767851008 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.616764E-01 | grad norm: 0.358 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 1853/ 3100 | consumed samples: 3794944 | consumed tokens: 7772045312 | elapsed time per iteration (s): 140.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.482288E-01 | grad norm: 0.653 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.553 | TFLOPs: 148.56 | -[default7]: iteration 1854/ 3100 | consumed samples: 3796992 | consumed tokens: 7776239616 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.541169E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 1855/ 3100 | consumed samples: 3799040 | consumed tokens: 7780433920 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.580237E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 1856/ 3100 | consumed samples: 3801088 | consumed tokens: 7784628224 | elapsed time per iteration (s): 141.13 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.545592E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.511 | TFLOPs: 148.14 | -[default7]: iteration 1857/ 3100 | consumed samples: 3803136 | consumed tokens: 7788822528 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.704148E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 1858/ 3100 | consumed samples: 3805184 | consumed tokens: 7793016832 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.598988E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 1859/ 3100 | consumed samples: 3807232 | consumed tokens: 7797211136 | elapsed time per iteration (s): 141.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.497770E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.486 | TFLOPs: 147.88 | -[default7]: iteration 1860/ 3100 | consumed samples: 3809280 | consumed tokens: 7801405440 | elapsed time per iteration (s): 141.12 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.620969E-01 | grad norm: 1.072 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.512 | TFLOPs: 148.15 | -[default7]: iteration 1861/ 3100 | consumed samples: 3811328 | consumed tokens: 7805599744 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.634352E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 1862/ 3100 | consumed samples: 3813376 | consumed tokens: 7809794048 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.569721E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 1863/ 3100 | consumed samples: 3815424 | consumed tokens: 7813988352 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.549413E-01 | grad norm: 0.342 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1864/ 3100 | consumed samples: 3817472 | consumed tokens: 7818182656 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.653989E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1865/ 3100 | consumed samples: 3819520 | consumed tokens: 7822376960 | elapsed time per iteration (s): 140.38 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.717041E-01 | grad norm: 2.026 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.589 | TFLOPs: 148.93 | -[default7]: iteration 1866/ 3100 | consumed samples: 3821568 | consumed tokens: 7826571264 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.587199E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.54 | -[default7]: iteration 1867/ 3100 | consumed samples: 3823616 | consumed tokens: 7830765568 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.494082E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1868/ 3100 | consumed samples: 3825664 | consumed tokens: 7834959872 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.551605E-01 | grad norm: 0.459 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.82 | -[default7]: iteration 1869/ 3100 | consumed samples: 3827712 | consumed tokens: 7839154176 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.683586E-01 | grad norm: 0.474 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1870/ 3100 | consumed samples: 3829760 | consumed tokens: 7843348480 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.700357E-01 | grad norm: 0.370 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 1871/ 3100 | consumed samples: 3831808 | consumed tokens: 7847542784 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.733998E-01 | grad norm: 0.667 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 1872/ 3100 | consumed samples: 3833856 | consumed tokens: 7851737088 | elapsed time per iteration (s): 140.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.472208E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.547 | TFLOPs: 148.50 | -[default7]: iteration 1873/ 3100 | consumed samples: 3835904 | consumed tokens: 7855931392 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.709525E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 1874/ 3100 | consumed samples: 3837952 | consumed tokens: 7860125696 | elapsed time per iteration (s): 140.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.621704E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.578 | TFLOPs: 148.82 | -[default7]: iteration 1875/ 3100 | consumed samples: 3840000 | consumed tokens: 7864320000 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.623801E-01 | grad norm: 0.422 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 1876/ 3100 | consumed samples: 3842048 | consumed tokens: 7868514304 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.657164E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1877/ 3100 | consumed samples: 3844096 | consumed tokens: 7872708608 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.722374E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 1878/ 3100 | consumed samples: 3846144 | consumed tokens: 7876902912 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.635043E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 1879/ 3100 | consumed samples: 3848192 | consumed tokens: 7881097216 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.677811E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1880/ 3100 | consumed samples: 3850240 | consumed tokens: 7885291520 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.565700E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 1881/ 3100 | consumed samples: 3852288 | consumed tokens: 7889485824 | elapsed time per iteration (s): 140.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.625780E-01 | grad norm: 0.352 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.565 | TFLOPs: 148.69 | -[default7]: iteration 1882/ 3100 | consumed samples: 3854336 | consumed tokens: 7893680128 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.627013E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 1883/ 3100 | consumed samples: 3856384 | consumed tokens: 7897874432 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.617642E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 1884/ 3100 | consumed samples: 3858432 | consumed tokens: 7902068736 | elapsed time per iteration (s): 140.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.576193E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.583 | TFLOPs: 148.87 | -[default7]: iteration 1885/ 3100 | consumed samples: 3860480 | consumed tokens: 7906263040 | elapsed time per iteration (s): 140.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.505342E-01 | grad norm: 0.370 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.556 | TFLOPs: 148.59 | -[default7]: iteration 1886/ 3100 | consumed samples: 3862528 | consumed tokens: 7910457344 | elapsed time per iteration (s): 141.31 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.781863E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.493 | TFLOPs: 147.95 | -[default7]: iteration 1887/ 3100 | consumed samples: 3864576 | consumed tokens: 7914651648 | elapsed time per iteration (s): 141.11 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.635764E-01 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.513 | TFLOPs: 148.16 | -[default7]: iteration 1888/ 3100 | consumed samples: 3866624 | consumed tokens: 7918845952 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.544065E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 1889/ 3100 | consumed samples: 3868672 | consumed tokens: 7923040256 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.654819E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1890/ 3100 | consumed samples: 3870720 | consumed tokens: 7927234560 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.600732E-01 | grad norm: 0.365 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 1891/ 3100 | consumed samples: 3872768 | consumed tokens: 7931428864 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.506446E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 1892/ 3100 | consumed samples: 3874816 | consumed tokens: 7935623168 | elapsed time per iteration (s): 139.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.637157E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.652 | TFLOPs: 149.57 | -[default7]: iteration 1893/ 3100 | consumed samples: 3876864 | consumed tokens: 7939817472 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.546583E-01 | grad norm: 0.802 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 1894/ 3100 | consumed samples: 3878912 | consumed tokens: 7944011776 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.749362E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1895/ 3100 | consumed samples: 3880960 | consumed tokens: 7948206080 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.454613E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 1896/ 3100 | consumed samples: 3883008 | consumed tokens: 7952400384 | elapsed time per iteration (s): 141.11 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.531216E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.513 | TFLOPs: 148.16 | -[default7]: iteration 1897/ 3100 | consumed samples: 3885056 | consumed tokens: 7956594688 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.601908E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 1898/ 3100 | consumed samples: 3887104 | consumed tokens: 7960788992 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.452191E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1899/ 3100 | consumed samples: 3889152 | consumed tokens: 7964983296 | elapsed time per iteration (s): 141.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.606978E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 1900/ 3100 | consumed samples: 3891200 | consumed tokens: 7969177600 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.560925E-01 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 1901/ 3100 | consumed samples: 3893248 | consumed tokens: 7973371904 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.477130E-01 | grad norm: 0.352 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.48 | -[default7]: iteration 1902/ 3100 | consumed samples: 3895296 | consumed tokens: 7977566208 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.669245E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 1903/ 3100 | consumed samples: 3897344 | consumed tokens: 7981760512 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.685593E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 1904/ 3100 | consumed samples: 3899392 | consumed tokens: 7985954816 | elapsed time per iteration (s): 140.13 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.542022E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.615 | TFLOPs: 149.19 | -[default7]: iteration 1905/ 3100 | consumed samples: 3901440 | consumed tokens: 7990149120 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.563134E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1906/ 3100 | consumed samples: 3903488 | consumed tokens: 7994343424 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.612566E-01 | grad norm: 0.415 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 1907/ 3100 | consumed samples: 3905536 | consumed tokens: 7998537728 | elapsed time per iteration (s): 141.31 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.545314E-01 | grad norm: 0.589 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.493 | TFLOPs: 147.95 | -[default7]: iteration 1908/ 3100 | consumed samples: 3907584 | consumed tokens: 8002732032 | elapsed time per iteration (s): 139.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.601951E-01 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.641 | TFLOPs: 149.47 | -[default7]: iteration 1909/ 3100 | consumed samples: 3909632 | consumed tokens: 8006926336 | elapsed time per iteration (s): 141.05 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.651109E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.520 | TFLOPs: 148.22 | -[default7]: iteration 1910/ 3100 | consumed samples: 3911680 | consumed tokens: 8011120640 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.636920E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1911/ 3100 | consumed samples: 3913728 | consumed tokens: 8015314944 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.481978E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 1912/ 3100 | consumed samples: 3915776 | consumed tokens: 8019509248 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.643389E-01 | grad norm: 0.453 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 1913/ 3100 | consumed samples: 3917824 | consumed tokens: 8023703552 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.455198E-01 | grad norm: 0.458 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1914/ 3100 | consumed samples: 3919872 | consumed tokens: 8027897856 | elapsed time per iteration (s): 140.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.530829E-01 | grad norm: 0.448 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.554 | TFLOPs: 148.58 | -[default7]: iteration 1915/ 3100 | consumed samples: 3921920 | consumed tokens: 8032092160 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.484314E-01 | grad norm: 0.615 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 1916/ 3100 | consumed samples: 3923968 | consumed tokens: 8036286464 | elapsed time per iteration (s): 141.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.542753E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.491 | TFLOPs: 147.93 | -[default7]: iteration 1917/ 3100 | consumed samples: 3926016 | consumed tokens: 8040480768 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.557962E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 1918/ 3100 | consumed samples: 3928064 | consumed tokens: 8044675072 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.528294E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 1919/ 3100 | consumed samples: 3930112 | consumed tokens: 8048869376 | elapsed time per iteration (s): 140.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.548591E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.549 | TFLOPs: 148.52 | -[default7]: iteration 1920/ 3100 | consumed samples: 3932160 | consumed tokens: 8053063680 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.589283E-01 | grad norm: 0.465 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 1921/ 3100 | consumed samples: 3934208 | consumed tokens: 8057257984 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.518677E-01 | grad norm: 0.638 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1922/ 3100 | consumed samples: 3936256 | consumed tokens: 8061452288 | elapsed time per iteration (s): 141.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.487674E-01 | grad norm: 0.431 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.428 | TFLOPs: 147.29 | -[default7]: iteration 1923/ 3100 | consumed samples: 3938304 | consumed tokens: 8065646592 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.557296E-01 | grad norm: 0.573 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 1924/ 3100 | consumed samples: 3940352 | consumed tokens: 8069840896 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.625216E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 1925/ 3100 | consumed samples: 3942400 | consumed tokens: 8074035200 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.699496E-01 | grad norm: 0.395 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 1926/ 3100 | consumed samples: 3944448 | consumed tokens: 8078229504 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.595911E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.31 | -[default7]: iteration 1927/ 3100 | consumed samples: 3946496 | consumed tokens: 8082423808 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.586116E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.72 | -[default7]: iteration 1928/ 3100 | consumed samples: 3948544 | consumed tokens: 8086618112 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.461233E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 1929/ 3100 | consumed samples: 3950592 | consumed tokens: 8090812416 | elapsed time per iteration (s): 141.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.626790E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.35 | -[default7]: iteration 1930/ 3100 | consumed samples: 3952640 | consumed tokens: 8095006720 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.532439E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1931/ 3100 | consumed samples: 3954688 | consumed tokens: 8099201024 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.627412E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 1932/ 3100 | consumed samples: 3956736 | consumed tokens: 8103395328 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.547193E-01 | grad norm: 0.358 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 1933/ 3100 | consumed samples: 3958784 | consumed tokens: 8107589632 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.457461E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 1934/ 3100 | consumed samples: 3960832 | consumed tokens: 8111783936 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.509292E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 1935/ 3100 | consumed samples: 3962880 | consumed tokens: 8115978240 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.578297E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 1936/ 3100 | consumed samples: 3964928 | consumed tokens: 8120172544 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.517733E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1937/ 3100 | consumed samples: 3966976 | consumed tokens: 8124366848 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.532645E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1938/ 3100 | consumed samples: 3969024 | consumed tokens: 8128561152 | elapsed time per iteration (s): 141.14 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.374724E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.510 | TFLOPs: 148.13 | -[default7]: iteration 1939/ 3100 | consumed samples: 3971072 | consumed tokens: 8132755456 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.580505E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 1940/ 3100 | consumed samples: 3973120 | consumed tokens: 8136949760 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.465802E-01 | grad norm: 0.339 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1941/ 3100 | consumed samples: 3975168 | consumed tokens: 8141144064 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.676271E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 1942/ 3100 | consumed samples: 3977216 | consumed tokens: 8145338368 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.533104E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 1943/ 3100 | consumed samples: 3979264 | consumed tokens: 8149532672 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.387747E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 1944/ 3100 | consumed samples: 3981312 | consumed tokens: 8153726976 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.449133E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1945/ 3100 | consumed samples: 3983360 | consumed tokens: 8157921280 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.501870E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 1946/ 3100 | consumed samples: 3985408 | consumed tokens: 8162115584 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.276759E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 1947/ 3100 | consumed samples: 3987456 | consumed tokens: 8166309888 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.596231E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 1948/ 3100 | consumed samples: 3989504 | consumed tokens: 8170504192 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.463832E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 1949/ 3100 | consumed samples: 3991552 | consumed tokens: 8174698496 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.521066E-01 | grad norm: 0.525 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.499 | TFLOPs: 148.02 | -[default7]: iteration 1950/ 3100 | consumed samples: 3993600 | consumed tokens: 8178892800 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.520978E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1951/ 3100 | consumed samples: 3995648 | consumed tokens: 8183087104 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.527758E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 1952/ 3100 | consumed samples: 3997696 | consumed tokens: 8187281408 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.519792E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 1953/ 3100 | consumed samples: 3999744 | consumed tokens: 8191475712 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.543339E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 1954/ 3100 | consumed samples: 4001792 | consumed tokens: 8195670016 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.508970E-01 | grad norm: 0.445 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 1955/ 3100 | consumed samples: 4003840 | consumed tokens: 8199864320 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.484722E-01 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1956/ 3100 | consumed samples: 4005888 | consumed tokens: 8204058624 | elapsed time per iteration (s): 140.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.545221E-01 | grad norm: 0.517 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.530 | TFLOPs: 148.33 | -[default7]: iteration 1957/ 3100 | consumed samples: 4007936 | consumed tokens: 8208252928 | elapsed time per iteration (s): 140.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.412981E-01 | grad norm: 0.358 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.590 | TFLOPs: 148.95 | -[default7]: iteration 1958/ 3100 | consumed samples: 4009984 | consumed tokens: 8212447232 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.483351E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.80 | -[default7]: iteration 1959/ 3100 | consumed samples: 4012032 | consumed tokens: 8216641536 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.493352E-01 | grad norm: 0.350 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1960/ 3100 | consumed samples: 4014080 | consumed tokens: 8220835840 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.450306E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 1961/ 3100 | consumed samples: 4016128 | consumed tokens: 8225030144 | elapsed time per iteration (s): 141.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.288172E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.487 | TFLOPs: 147.89 | -[default7]: iteration 1962/ 3100 | consumed samples: 4018176 | consumed tokens: 8229224448 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.454083E-01 | grad norm: 0.339 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 1963/ 3100 | consumed samples: 4020224 | consumed tokens: 8233418752 | elapsed time per iteration (s): 139.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.572433E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.661 | TFLOPs: 149.67 | -[default7]: iteration 1964/ 3100 | consumed samples: 4022272 | consumed tokens: 8237613056 | elapsed time per iteration (s): 140.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.568920E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.556 | TFLOPs: 148.59 | -[default7]: iteration 1965/ 3100 | consumed samples: 4024320 | consumed tokens: 8241807360 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.525808E-01 | grad norm: 0.332 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 1966/ 3100 | consumed samples: 4026368 | consumed tokens: 8246001664 | elapsed time per iteration (s): 141.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.501242E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.81 | -[default7]: iteration 1967/ 3100 | consumed samples: 4028416 | consumed tokens: 8250195968 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.300546E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 1968/ 3100 | consumed samples: 4030464 | consumed tokens: 8254390272 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.561446E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 1969/ 3100 | consumed samples: 4032512 | consumed tokens: 8258584576 | elapsed time per iteration (s): 140.21 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.464148E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.607 | TFLOPs: 149.11 | -[default7]: iteration 1970/ 3100 | consumed samples: 4034560 | consumed tokens: 8262778880 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.373881E-01 | grad norm: 0.347 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 1971/ 3100 | consumed samples: 4036608 | consumed tokens: 8266973184 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.555905E-01 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 1972/ 3100 | consumed samples: 4038656 | consumed tokens: 8271167488 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.425718E-01 | grad norm: 0.521 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 1973/ 3100 | consumed samples: 4040704 | consumed tokens: 8275361792 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.595984E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 1974/ 3100 | consumed samples: 4042752 | consumed tokens: 8279556096 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.539333E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 1975/ 3100 | consumed samples: 4044800 | consumed tokens: 8283750400 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.426130E-01 | grad norm: 0.544 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 1976/ 3100 | consumed samples: 4046848 | consumed tokens: 8287944704 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.474377E-01 | grad norm: 0.378 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 1977/ 3100 | consumed samples: 4048896 | consumed tokens: 8292139008 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.597548E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 1978/ 3100 | consumed samples: 4050944 | consumed tokens: 8296333312 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.552162E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 1979/ 3100 | consumed samples: 4052992 | consumed tokens: 8300527616 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.455899E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 1980/ 3100 | consumed samples: 4055040 | consumed tokens: 8304721920 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.538893E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 1981/ 3100 | consumed samples: 4057088 | consumed tokens: 8308916224 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.468007E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 1982/ 3100 | consumed samples: 4059136 | consumed tokens: 8313110528 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.347372E-01 | grad norm: 0.341 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 1983/ 3100 | consumed samples: 4061184 | consumed tokens: 8317304832 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.385294E-01 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 1984/ 3100 | consumed samples: 4063232 | consumed tokens: 8321499136 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.557626E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 1985/ 3100 | consumed samples: 4065280 | consumed tokens: 8325693440 | elapsed time per iteration (s): 141.34 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.607646E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.490 | TFLOPs: 147.92 | -[default7]: iteration 1986/ 3100 | consumed samples: 4067328 | consumed tokens: 8329887744 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.571321E-01 | grad norm: 0.398 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 1987/ 3100 | consumed samples: 4069376 | consumed tokens: 8334082048 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.610996E-01 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.78 | -[default7]: iteration 1988/ 3100 | consumed samples: 4071424 | consumed tokens: 8338276352 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.484615E-01 | grad norm: 0.347 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 1989/ 3100 | consumed samples: 4073472 | consumed tokens: 8342470656 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.529917E-01 | grad norm: 0.334 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 1990/ 3100 | consumed samples: 4075520 | consumed tokens: 8346664960 | elapsed time per iteration (s): 141.16 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.413273E-01 | grad norm: 0.496 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.508 | TFLOPs: 148.10 | -[default7]: iteration 1991/ 3100 | consumed samples: 4077568 | consumed tokens: 8350859264 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.434141E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default0]:saving checkpoint at iteration 1992 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-11 04:34:19,741] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1992 is begin to save! -[default7]: iteration 1992/ 3100 | consumed samples: 4079616 | consumed tokens: 8355053568 | elapsed time per iteration (s): 140.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.415816E-01 | grad norm: 0.352 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.575 | TFLOPs: 148.79 | -[default0]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_12-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_34-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,815] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_47-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,833] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_27-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_36-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_48-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_62-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_42-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_08-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,849] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_41-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_04-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,838] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_01-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_64-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_39-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_21-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_20-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_61-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_53-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_69-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_24-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_49-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_17-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_33-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_23-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,834] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_71-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_72-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_16-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_15-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_10-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_54-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_67-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_45-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_37-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_58-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_40-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_07-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,815] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_46-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_05-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_59-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_31-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_19-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_66-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_11-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_56-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_03-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_50-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,834] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_70-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_52-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_32-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_68-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_13-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_22-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_60-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,833] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_26-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_30-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_55-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_63-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_18-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_35-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_06-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_44-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,842] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_71_model_states.pt... -[default4]:[2022-09-11 04:34:19,881] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_71_model_states.pt. -[default0]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_38-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_51-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_43-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_57-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,836] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_09-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_25-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_14-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,943] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_29-model_00-model_states.pt... -[default4]:[2022-09-11 04:34:19,848] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_65-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:19,943] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_28-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:23,242] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_46-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_44_model_states.pt... -[default0]:[2022-09-11 04:34:23,250] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_44_model_states.pt. -[default4]:[2022-09-11 04:34:23,369] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_71-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,370] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_69_model_states.pt... -[default0]:[2022-09-11 04:34:23,289] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_72-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,290] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_74-model_00-model_states.pt... -[default0]:[2022-09-11 04:34:23,292] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_74-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_70_model_states.pt... -[default0]:[2022-09-11 04:34:23,294] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_70_model_states.pt. -[default0]:[2022-09-11 04:34:23,398] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_66-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,398] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_64_model_states.pt... -[default0]:[2022-09-11 04:34:23,383] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_26-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,383] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_24_model_states.pt... -[default0]:[2022-09-11 04:34:23,392] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_24_model_states.pt. -[default0]:[2022-09-11 04:34:23,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_12-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,385] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_10_model_states.pt... -[default0]:[2022-09-11 04:34:23,391] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_10_model_states.pt. -[default4]:[2022-09-11 04:34:23,432] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_27-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,432] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_25_model_states.pt... -[default4]:[2022-09-11 04:34:23,434] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_25_model_states.pt. -[default0]:[2022-09-11 04:34:23,445] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_48-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,445] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_46_model_states.pt... -[default0]:[2022-09-11 04:34:23,447] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_46_model_states.pt. -[default4]:[2022-09-11 04:34:23,411] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_49-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,411] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_47_model_states.pt... -[default4]:[2022-09-11 04:34:23,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_47_model_states.pt. -[default4]:[2022-09-11 04:34:23,380] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_69_model_states.pt. -[default4]:[2022-09-11 04:34:23,411] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_05-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,411] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_03_model_states.pt... -[default4]:[2022-09-11 04:34:23,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_03_model_states.pt. -[default0]:[2022-09-11 04:34:23,404] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_64_model_states.pt. -[default4]:[2022-09-11 04:34:23,494] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_13-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,494] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_11_model_states.pt... -[default4]:[2022-09-11 04:34:23,496] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_11_model_states.pt. -[default0]:[2022-09-11 04:34:23,520] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_30-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,521] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_28_model_states.pt... -[default4]:[2022-09-11 04:34:23,515] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_47-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_45_model_states.pt... -[default4]:[2022-09-11 04:34:23,534] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_45_model_states.pt. -[default4]:[2022-09-11 04:34:23,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_61-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,547] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_59_model_states.pt... -[default4]:[2022-09-11 04:34:23,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_23-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,503] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_21_model_states.pt... -[default4]:[2022-09-11 04:34:23,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_21_model_states.pt. -[default4]:[2022-09-11 04:34:23,563] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_31-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,563] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_29_model_states.pt... -[default4]:[2022-09-11 04:34:23,566] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_29_model_states.pt. -[default0]:[2022-09-11 04:34:23,567] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_70-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,567] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_68_model_states.pt... -[default0]:[2022-09-11 04:34:23,570] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_68_model_states.pt. -[default0]:[2022-09-11 04:34:23,547] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_28_model_states.pt. -[default0]:[2022-09-11 04:34:23,617] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_34-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,618] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_32_model_states.pt... -[default0]:[2022-09-11 04:34:23,627] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_32_model_states.pt. -[default0]:[2022-09-11 04:34:23,638] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_04-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,638] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_02_model_states.pt... -[default0]:[2022-09-11 04:34:23,641] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_02_model_states.pt. -[default4]:[2022-09-11 04:34:23,571] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_59_model_states.pt. -[default4]:[2022-09-11 04:34:23,638] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_33-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,638] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_31_model_states.pt... -[default4]:[2022-09-11 04:34:23,644] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_31_model_states.pt. -[default0]:[2022-09-11 04:34:23,669] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_16-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,670] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_14_model_states.pt... -[default4]:[2022-09-11 04:34:23,659] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_19-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,660] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_17_model_states.pt... -[default4]:[2022-09-11 04:34:23,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_17_model_states.pt. -[default0]:[2022-09-11 04:34:23,664] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_56-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,665] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_54_model_states.pt... -[default0]:[2022-09-11 04:34:23,682] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_54_model_states.pt. -[default4]:[2022-09-11 04:34:23,698] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_03-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,698] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_01_model_states.pt... -[default0]:[2022-09-11 04:34:23,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_32-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,647] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_30_model_states.pt... -[default0]:[2022-09-11 04:34:23,650] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_30_model_states.pt. -[default0]:[2022-09-11 04:34:23,669] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_22-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,670] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_20_model_states.pt... -[default0]:[2022-09-11 04:34:23,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_20_model_states.pt. -[default0]:[2022-09-11 04:34:23,638] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_60-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,639] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_58_model_states.pt... -[default0]:[2022-09-11 04:34:23,641] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_58_model_states.pt. -[default4]:[2022-09-11 04:34:23,722] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_63-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,723] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_61_model_states.pt... -[default0]:[2022-09-11 04:34:23,704] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_18-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,705] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_16_model_states.pt... -[default0]:[2022-09-11 04:34:23,712] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_16_model_states.pt. -[default4]:[2022-09-11 04:34:23,671] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_35-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_33_model_states.pt... -[default4]:[2022-09-11 04:34:23,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_33_model_states.pt. -[default0]:[2022-09-11 04:34:23,668] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_06-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,668] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_04_model_states.pt... -[default0]:[2022-09-11 04:34:23,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_04_model_states.pt. -[default0]:[2022-09-11 04:34:23,706] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_38-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,706] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_36_model_states.pt... -[default0]:[2022-09-11 04:34:23,719] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_36_model_states.pt. -[default0]:[2022-09-11 04:34:23,655] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_44-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,656] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_42_model_states.pt... -[default0]:[2022-09-11 04:34:23,671] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_42_model_states.pt. -[default4]:[2022-09-11 04:34:23,701] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_09-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,702] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_07_model_states.pt... -[default4]:[2022-09-11 04:34:23,710] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_07_model_states.pt. -[default4]:[2022-09-11 04:34:23,702] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_69-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,702] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_67_model_states.pt... -[default4]:[2022-09-11 04:34:23,709] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_67_model_states.pt. -[default4]:[2022-09-11 04:34:23,707] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_17-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,708] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_15_model_states.pt... -[default4]:[2022-09-11 04:34:23,718] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_15_model_states.pt. -[default4]:[2022-09-11 04:34:23,708] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_21-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,708] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_19_model_states.pt... -[default4]:[2022-09-11 04:34:23,723] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_19_model_states.pt. -[default0]:[2022-09-11 04:34:23,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_14_model_states.pt. -[default0]:[2022-09-11 04:34:23,729] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_58-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,729] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_56_model_states.pt... -[default0]:[2022-09-11 04:34:23,742] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_56_model_states.pt. -[default4]:[2022-09-11 04:34:23,695] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_07-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,695] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_05_model_states.pt... -[default4]:[2022-09-11 04:34:23,697] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_05_model_states.pt. -[default4]:[2022-09-11 04:34:23,764] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_59-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,764] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_57_model_states.pt... -[default4]:[2022-09-11 04:34:23,773] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_57_model_states.pt. -[default4]:[2022-09-11 04:34:23,709] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_01_model_states.pt. -[default4]:[2022-09-11 04:34:23,729] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_61_model_states.pt. -[default4]:[2022-09-11 04:34:23,765] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_57-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,765] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_55_model_states.pt... -[default4]:[2022-09-11 04:34:23,768] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_55_model_states.pt. -[default0]:[2022-09-11 04:34:23,793] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_42-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,794] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_40_model_states.pt... -[default0]:[2022-09-11 04:34:23,824] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_40_model_states.pt. -[default0]:[2022-09-11 04:34:23,774] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_20-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,774] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_18_model_states.pt... -[default0]:[2022-09-11 04:34:23,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_18_model_states.pt. -[default4]:[2022-09-11 04:34:23,825] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_67-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,826] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_65_model_states.pt... -[default4]:[2022-09-11 04:34:23,829] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_65_model_states.pt. -[default4]:[2022-09-11 04:34:23,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_45-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_43_model_states.pt... -[default4]:[2022-09-11 04:34:23,843] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_43_model_states.pt. -[default4]:[2022-09-11 04:34:23,857] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_37-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,857] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_35_model_states.pt... -[default4]:[2022-09-11 04:34:23,866] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_35_model_states.pt. -[default0]:[2022-09-11 04:34:23,891] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_50-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,891] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_48_model_states.pt... -[default0]:[2022-09-11 04:34:23,901] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_48_model_states.pt. -[default0]:[2022-09-11 04:34:23,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_52-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,855] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_50_model_states.pt... -[default0]:[2022-09-11 04:34:23,863] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_50_model_states.pt. -[default0]:[2022-09-11 04:34:23,876] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_68-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,876] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_66_model_states.pt... -[default0]:[2022-09-11 04:34:23,879] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_66_model_states.pt. -[default4]:[2022-09-11 04:34:23,915] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_51-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,915] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_49_model_states.pt... -[default4]:[2022-09-11 04:34:23,917] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_49_model_states.pt. -[default4]:[2022-09-11 04:34:23,909] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_43-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,909] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_41_model_states.pt... -[default4]:[2022-09-11 04:34:23,911] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_41_model_states.pt. -[default4]:[2022-09-11 04:34:23,878] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_65-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,878] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_63_model_states.pt... -[default4]:[2022-09-11 04:34:23,881] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_63_model_states.pt. -[default0]:[2022-09-11 04:34:23,912] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_36-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,912] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_34_model_states.pt... -[default0]:[2022-09-11 04:34:23,914] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_34_model_states.pt. -[default0]:[2022-09-11 04:34:23,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_62-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,887] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_60_model_states.pt... -[default0]:[2022-09-11 04:34:23,890] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_60_model_states.pt. -[default0]:[2022-09-11 04:34:23,862] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_08-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,862] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_06_model_states.pt... -[default0]:[2022-09-11 04:34:23,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_06_model_states.pt. -[default0]:[2022-09-11 04:34:23,904] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_64-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,905] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_62_model_states.pt... -[default0]:[2022-09-11 04:34:23,907] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_62_model_states.pt. -[default4]:[2022-09-11 04:34:23,885] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_39-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,885] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_37_model_states.pt... -[default4]:[2022-09-11 04:34:23,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_37_model_states.pt. -[default4]:[2022-09-11 04:34:23,958] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_53-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,958] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_51_model_states.pt... -[default4]:[2022-09-11 04:34:23,960] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_51_model_states.pt. -[default0]:[2022-09-11 04:34:23,960] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_24-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_22_model_states.pt... -[default0]:[2022-09-11 04:34:23,962] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_22_model_states.pt. -[default4]:[2022-09-11 04:34:23,966] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_15-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,967] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_13_model_states.pt... -[default4]:[2022-09-11 04:34:23,971] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_13_model_states.pt. -[default0]:[2022-09-11 04:34:23,951] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_40-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,951] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_38_model_states.pt... -[default0]:[2022-09-11 04:34:23,954] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_38_model_states.pt. -[default4]:[2022-09-11 04:34:23,955] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_55-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,955] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_53_model_states.pt... -[default4]:[2022-09-11 04:34:23,957] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_53_model_states.pt. -[default4]:[2022-09-11 04:34:23,971] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_25-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,972] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_23_model_states.pt... -[default4]:[2022-09-11 04:34:23,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_23_model_states.pt. -[default0]:[2022-09-11 04:34:23,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_14-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,980] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_12_model_states.pt... -[default0]:[2022-09-11 04:34:23,982] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_12_model_states.pt. -[default4]:[2022-09-11 04:34:24,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_29-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:24,005] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_27_model_states.pt... -[default4]:[2022-09-11 04:34:24,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_27_model_states.pt. -[default4]:[2022-09-11 04:34:23,979] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_41-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:23,979] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_39_model_states.pt... -[default4]:[2022-09-11 04:34:23,982] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_39_model_states.pt. -[default0]:[2022-09-11 04:34:24,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_10-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:24,005] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_08_model_states.pt... -[default0]:[2022-09-11 04:34:24,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_08_model_states.pt. -[default0]:[2022-09-11 04:34:23,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_54-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:23,993] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_52_model_states.pt... -[default0]:[2022-09-11 04:34:23,995] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_52_model_states.pt. -[default0]:[2022-09-11 04:34:24,057] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_28-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:24,058] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_26_model_states.pt... -[default0]:[2022-09-11 04:34:24,059] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_26_model_states.pt. -[default4]:[2022-09-11 04:34:24,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_11-model_00-model_states.pt. -[default4]:[2022-09-11 04:34:24,026] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_09_model_states.pt... -[default4]:[2022-09-11 04:34:24,027] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_09_model_states.pt. -[default0]:[2022-09-11 04:34:24,853] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/layer_01-model_00-model_states.pt. -[default0]:[2022-09-11 04:34:24,854] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_00_model_states.pt -[default0]:[2022-09-11 04:34:24,854] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_00_model_states.pt... -[default0]:[2022-09-11 04:34:24,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/mp_rank_00_model_states.pt. -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default5]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default4]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default1]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default6]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default2]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default0]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default3]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default7]:[2022-09-11 04:34:24,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default5]:[2022-09-11 04:34:33,471] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-11 04:34:33,472] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default4]:[2022-09-11 04:34:33,512] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-11 04:34:33,512] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default7]:[2022-09-11 04:34:33,626] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-11 04:34:33,626] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default2]:[2022-09-11 04:34:33,705] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-11 04:34:33,705] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default5]:[2022-09-11 04:34:33,912] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-11 04:34:33,912] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default3]:[2022-09-11 04:34:34,094] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-11 04:34:34,094] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default3]:[2022-09-11 04:34:34,202] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-11 04:34:34,202] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default0]:[2022-09-11 04:34:34,322] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-11 04:34:34,322] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default0]:[2022-09-11 04:34:34,270] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-11 04:34:34,270] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default3]:[2022-09-11 04:34:34,334] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-11 04:34:34,335] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default0]:[2022-09-11 04:34:34,369] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-11 04:34:34,370] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default1]:[2022-09-11 04:34:34,413] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-11 04:34:34,414] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default1]:[2022-09-11 04:34:34,366] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-11 04:34:34,366] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default2]:[2022-09-11 04:34:34,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-11 04:34:34,418] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default4]:[2022-09-11 04:34:34,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-11 04:34:34,417] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default7]:[2022-09-11 04:34:34,474] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-11 04:34:34,474] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default3]:[2022-09-11 04:34:34,538] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-11 04:34:34,539] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default1]:[2022-09-11 04:34:34,504] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-11 04:34:34,504] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default0]:[2022-09-11 04:34:34,558] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-11 04:34:34,558] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default2]:[2022-09-11 04:34:34,587] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-11 04:34:34,587] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default6]:[2022-09-11 04:34:34,627] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-11 04:34:34,627] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default0]:[2022-09-11 04:34:34,678] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-11 04:34:34,678] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default0]:[2022-09-11 04:34:34,702] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-11 04:34:34,703] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default5]:[2022-09-11 04:34:34,678] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-11 04:34:34,679] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default1]:[2022-09-11 04:34:34,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-11 04:34:34,757] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default4]:[2022-09-11 04:34:34,748] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-11 04:34:34,748] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default6]:[2022-09-11 04:34:34,798] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-11 04:34:34,798] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default6]:[2022-09-11 04:34:34,753] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-11 04:34:34,753] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default5]:[2022-09-11 04:34:34,811] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-11 04:34:34,811] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default3]:[2022-09-11 04:34:34,824] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-11 04:34:34,824] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default6]:[2022-09-11 04:34:34,805] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-11 04:34:34,805] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default0]:[2022-09-11 04:34:34,886] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-11 04:34:34,886] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default4]:[2022-09-11 04:34:34,964] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-11 04:34:34,964] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default0]:[2022-09-11 04:34:35,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-11 04:34:35,004] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default1]:[2022-09-11 04:34:35,065] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-11 04:34:35,065] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default2]:[2022-09-11 04:34:35,149] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-11 04:34:35,149] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default3]:[2022-09-11 04:34:35,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-11 04:34:35,221] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default5]:[2022-09-11 04:34:35,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-11 04:34:35,284] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default7]:[2022-09-11 04:34:35,295] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-11 04:34:35,295] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default4]:[2022-09-11 04:34:35,264] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-11 04:34:35,265] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default7]:[2022-09-11 04:34:35,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-11 04:34:35,310] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default6]:[2022-09-11 04:34:35,279] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-11 04:34:35,279] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default0]:[2022-09-11 04:34:35,302] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-11 04:34:35,302] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default5]:[2022-09-11 04:34:35,354] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-11 04:34:35,354] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default7]:[2022-09-11 04:34:35,403] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-11 04:34:35,404] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default3]:[2022-09-11 04:34:35,484] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-11 04:34:35,484] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default7]:[2022-09-11 04:34:35,441] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-11 04:34:35,441] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default0]:[2022-09-11 04:34:35,472] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-11 04:34:35,472] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default4]:[2022-09-11 04:34:35,466] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-11 04:34:35,467] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default6]:[2022-09-11 04:34:35,526] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-11 04:34:35,527] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default2]:[2022-09-11 04:34:35,541] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-11 04:34:35,541] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default4]:[2022-09-11 04:34:35,614] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-11 04:34:35,614] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default2]:[2022-09-11 04:34:35,610] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-11 04:34:35,610] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default1]:[2022-09-11 04:34:35,660] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-11 04:34:35,660] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default2]:[2022-09-11 04:34:35,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-11 04:34:35,728] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default4]:[2022-09-11 04:34:35,702] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-11 04:34:35,703] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default6]:[2022-09-11 04:34:35,745] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-11 04:34:35,745] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default6]:[2022-09-11 04:34:35,701] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-11 04:34:35,701] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default3]:[2022-09-11 04:34:35,704] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-11 04:34:35,705] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default4]:[2022-09-11 04:34:35,693] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-11 04:34:35,693] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default3]:[2022-09-11 04:34:35,859] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-11 04:34:35,859] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default0]:[2022-09-11 04:34:35,815] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-11 04:34:35,815] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default2]:[2022-09-11 04:34:35,883] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-11 04:34:35,883] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default0]:[2022-09-11 04:34:35,873] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-11 04:34:35,873] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default7]:[2022-09-11 04:34:35,925] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-11 04:34:35,925] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default0]:[2022-09-11 04:34:35,946] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-11 04:34:35,946] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default6]:[2022-09-11 04:34:35,919] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-11 04:34:35,919] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default3]:[2022-09-11 04:34:35,981] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-11 04:34:35,981] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default0]:[2022-09-11 04:34:35,989] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-11 04:34:35,990] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default0]:[2022-09-11 04:34:35,951] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-11 04:34:35,951] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default4]:[2022-09-11 04:34:35,991] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-11 04:34:35,991] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default2]:[2022-09-11 04:34:36,087] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-11 04:34:36,087] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default7]:[2022-09-11 04:34:36,019] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-11 04:34:36,019] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default7]:[2022-09-11 04:34:36,099] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-11 04:34:36,100] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default6]:[2022-09-11 04:34:36,140] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-11 04:34:36,140] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default2]:[2022-09-11 04:34:36,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-11 04:34:36,046] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default5]:[2022-09-11 04:34:36,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-11 04:34:36,089] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default1]:[2022-09-11 04:34:36,170] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-11 04:34:36,170] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default0]:[2022-09-11 04:34:36,092] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-11 04:34:36,092] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default1]:[2022-09-11 04:34:36,153] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-11 04:34:36,153] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default5]:[2022-09-11 04:34:36,211] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-11 04:34:36,211] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default7]:[2022-09-11 04:34:36,154] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-11 04:34:36,155] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default0]:[2022-09-11 04:34:36,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-11 04:34:36,203] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default1]:[2022-09-11 04:34:36,254] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-11 04:34:36,254] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default6]:[2022-09-11 04:34:36,179] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-11 04:34:36,179] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default3]:[2022-09-11 04:34:36,230] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-11 04:34:36,231] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default7]:[2022-09-11 04:34:36,247] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-11 04:34:36,248] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default3]:[2022-09-11 04:34:36,316] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-11 04:34:36,316] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default1]:[2022-09-11 04:34:36,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-11 04:34:36,284] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default3]:[2022-09-11 04:34:36,366] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-11 04:34:36,366] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default7]:[2022-09-11 04:34:36,388] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-11 04:34:36,388] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default7]:[2022-09-11 04:34:36,374] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-11 04:34:36,374] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default5]:[2022-09-11 04:34:36,439] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-11 04:34:36,439] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default3]:[2022-09-11 04:34:36,467] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-11 04:34:36,467] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default0]:[2022-09-11 04:34:36,431] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-11 04:34:36,431] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default7]:[2022-09-11 04:34:36,477] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-11 04:34:36,477] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default4]:[2022-09-11 04:34:36,489] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-11 04:34:36,489] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default4]:[2022-09-11 04:34:36,574] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-11 04:34:36,575] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default4]:[2022-09-11 04:34:36,531] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-11 04:34:36,531] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default5]:[2022-09-11 04:34:36,524] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-11 04:34:36,524] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default2]:[2022-09-11 04:34:36,597] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-11 04:34:36,597] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default7]:[2022-09-11 04:34:36,568] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-11 04:34:36,568] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default6]:[2022-09-11 04:34:36,586] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-11 04:34:36,586] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default2]:[2022-09-11 04:34:36,598] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-11 04:34:36,598] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default6]:[2022-09-11 04:34:36,577] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-11 04:34:36,578] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default4]:[2022-09-11 04:34:36,634] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-11 04:34:36,634] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default3]:[2022-09-11 04:34:36,658] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-11 04:34:36,658] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default2]:[2022-09-11 04:34:36,590] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-11 04:34:36,590] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default5]:[2022-09-11 04:34:36,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-11 04:34:36,615] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default1]:[2022-09-11 04:34:36,617] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-11 04:34:36,617] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default1]:[2022-09-11 04:34:36,619] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-11 04:34:36,619] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default2]:[2022-09-11 04:34:36,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-11 04:34:36,685] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default2]:[2022-09-11 04:34:36,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-11 04:34:36,700] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default5]:[2022-09-11 04:34:36,649] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-11 04:34:36,649] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default5]:[2022-09-11 04:34:36,665] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-11 04:34:36,665] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default6]:[2022-09-11 04:34:36,726] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-11 04:34:36,727] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default3]:[2022-09-11 04:34:36,662] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-11 04:34:36,662] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default6]:[2022-09-11 04:34:36,668] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-11 04:34:36,668] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default7]:[2022-09-11 04:34:36,698] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-11 04:34:36,698] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default5]:[2022-09-11 04:34:36,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-11 04:34:36,784] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default3]:[2022-09-11 04:34:36,737] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-11 04:34:36,737] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default3]:[2022-09-11 04:34:36,798] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-11 04:34:36,798] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default5]:[2022-09-11 04:34:36,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-11 04:34:36,808] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default1]:[2022-09-11 04:34:36,768] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-11 04:34:36,768] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default1]:[2022-09-11 04:34:36,750] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-11 04:34:36,750] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default2]:[2022-09-11 04:34:36,845] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default6]:[2022-09-11 04:34:36,866] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-11 04:34:36,866] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default4]:[2022-09-11 04:34:36,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-11 04:34:36,861] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default1]:[2022-09-11 04:34:36,796] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-11 04:34:36,796] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default2]:[2022-09-11 04:34:36,886] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-11 04:34:36,886] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default4]:[2022-09-11 04:34:36,815] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-11 04:34:36,815] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default3]:[2022-09-11 04:34:36,826] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-11 04:34:36,826] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default6]:[2022-09-11 04:34:36,893] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-11 04:34:36,893] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default3]:[2022-09-11 04:34:36,860] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-11 04:34:36,860] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default2]:[2022-09-11 04:34:36,896] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-11 04:34:36,897] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default2]:[2022-09-11 04:34:36,845] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default7]:[2022-09-11 04:34:36,908] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-11 04:34:36,908] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default3]:[2022-09-11 04:34:36,898] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-11 04:34:36,898] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default7]:[2022-09-11 04:34:36,893] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-11 04:34:36,893] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default1]:[2022-09-11 04:34:36,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-11 04:34:36,973] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default4]:[2022-09-11 04:34:36,950] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-11 04:34:36,951] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default1]:[2022-09-11 04:34:36,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-11 04:34:36,947] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default5]:[2022-09-11 04:34:36,903] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-11 04:34:36,903] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default1]:[2022-09-11 04:34:36,915] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-11 04:34:36,916] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default2]:[2022-09-11 04:34:36,957] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-11 04:34:36,957] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default6]:[2022-09-11 04:34:36,943] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-11 04:34:36,943] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default0]:[2022-09-11 04:34:36,958] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-11 04:34:36,958] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default0]:[2022-09-11 04:34:36,949] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-11 04:34:36,949] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default3]:[2022-09-11 04:34:36,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-11 04:34:36,973] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default5]:[2022-09-11 04:34:36,970] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-11 04:34:36,970] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default6]:[2022-09-11 04:34:36,989] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-11 04:34:36,989] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default3]:[2022-09-11 04:34:37,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-11 04:34:37,026] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default7]:[2022-09-11 04:34:37,083] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-11 04:34:37,083] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default7]:[2022-09-11 04:34:37,078] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-11 04:34:37,078] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default6]:[2022-09-11 04:34:37,063] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-11 04:34:37,063] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default3]:[2022-09-11 04:34:37,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-11 04:34:37,043] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default7]:[2022-09-11 04:34:37,040] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-11 04:34:37,040] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default3]:[2022-09-11 04:34:37,123] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-11 04:34:37,124] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default1]:[2022-09-11 04:34:37,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-11 04:34:37,079] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default4]:[2022-09-11 04:34:37,126] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-11 04:34:37,126] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default2]:[2022-09-11 04:34:37,095] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-11 04:34:37,096] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default6]:[2022-09-11 04:34:37,083] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-11 04:34:37,083] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default6]:[2022-09-11 04:34:37,112] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-11 04:34:37,112] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default0]:[2022-09-11 04:34:37,181] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-11 04:34:37,182] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default3]:[2022-09-11 04:34:37,159] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-11 04:34:37,159] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default5]:[2022-09-11 04:34:37,187] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-11 04:34:37,187] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default3]:[2022-09-11 04:34:37,209] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-11 04:34:37,209] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default5]:[2022-09-11 04:34:37,166] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-11 04:34:37,166] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default5]:[2022-09-11 04:34:37,198] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-11 04:34:37,198] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default4]:[2022-09-11 04:34:37,224] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-11 04:34:37,224] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default2]:[2022-09-11 04:34:37,259] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-11 04:34:37,259] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default5]:[2022-09-11 04:34:37,229] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-11 04:34:37,230] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default4]:[2022-09-11 04:34:37,214] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-11 04:34:37,215] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default7]:[2022-09-11 04:34:37,240] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-11 04:34:37,240] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default0]:[2022-09-11 04:34:37,320] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-11 04:34:37,320] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default1]:[2022-09-11 04:34:37,271] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-11 04:34:37,272] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default4]:[2022-09-11 04:34:37,290] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-11 04:34:37,290] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default7]:[2022-09-11 04:34:37,353] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-11 04:34:37,354] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default1]:[2022-09-11 04:34:37,299] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-11 04:34:37,299] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default2]:[2022-09-11 04:34:37,369] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-11 04:34:37,369] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default2]:[2022-09-11 04:34:37,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-11 04:34:37,386] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default1]:[2022-09-11 04:34:37,346] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-11 04:34:37,346] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default1]:[2022-09-11 04:34:37,367] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-11 04:34:37,367] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default1]:[2022-09-11 04:34:37,429] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-11 04:34:37,430] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default4]:[2022-09-11 04:34:37,402] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-11 04:34:37,402] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default6]:[2022-09-11 04:34:37,401] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-11 04:34:37,401] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default0]:[2022-09-11 04:34:37,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-11 04:34:37,407] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default4]:[2022-09-11 04:34:37,426] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-11 04:34:37,426] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default1]:[2022-09-11 04:34:37,465] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-11 04:34:37,465] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default5]:[2022-09-11 04:34:37,466] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-11 04:34:37,466] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default6]:[2022-09-11 04:34:37,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-11 04:34:37,502] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default7]:[2022-09-11 04:34:37,449] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-11 04:34:37,449] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default2]:[2022-09-11 04:34:37,461] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-11 04:34:37,461] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default2]:[2022-09-11 04:34:37,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-11 04:34:37,561] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default5]:[2022-09-11 04:34:37,555] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-11 04:34:37,555] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default6]:[2022-09-11 04:34:37,569] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-11 04:34:37,569] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default0]:[2022-09-11 04:34:37,603] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-11 04:34:37,603] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default5]:[2022-09-11 04:34:37,568] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-11 04:34:37,568] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default7]:[2022-09-11 04:34:37,612] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-11 04:34:37,612] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default4]:[2022-09-11 04:34:37,580] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-11 04:34:37,580] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default6]:[2022-09-11 04:34:37,632] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-11 04:34:37,632] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default7]:[2022-09-11 04:34:37,722] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-11 04:34:37,722] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default5]:[2022-09-11 04:34:37,656] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-11 04:34:37,656] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default2]:[2022-09-11 04:34:37,751] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-11 04:34:37,751] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default5]:[2022-09-11 04:34:37,851] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-11 04:34:37,852] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default4]:[2022-09-11 04:34:37,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-11 04:34:37,910] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default7]:[2022-09-11 04:34:37,987] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-11 04:34:37,987] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default1]:[2022-09-11 04:34:38,078] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-11 04:34:38,078] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default0]:[2022-09-11 04:34:38,115] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-11 04:34:38,115] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default7]:[2022-09-11 04:34:38,137] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-11 04:34:38,137] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default3]:[2022-09-11 04:34:38,170] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-11 04:34:38,170] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default4]:[2022-09-11 04:34:38,234] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-11 04:34:38,234] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default6]:[2022-09-11 04:34:38,292] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-11 04:34:38,292] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default1]:[2022-09-11 04:34:38,361] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-11 04:34:38,361] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default6]:[2022-09-11 04:34:38,347] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-11 04:34:38,348] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default0]:[2022-09-11 04:34:38,422] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-11 04:34:38,422] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default5]:[2022-09-11 04:34:38,389] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-11 04:34:38,389] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default5]:[2022-09-11 04:34:38,545] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-11 04:34:38,546] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default5]:[2022-09-11 04:34:38,557] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-11 04:34:38,557] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default2]:[2022-09-11 04:34:38,659] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-11 04:34:38,659] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default4]:[2022-09-11 04:34:38,691] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-11 04:34:38,692] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default0]:[2022-09-11 04:34:38,758] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-11 04:34:38,758] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default7]:[2022-09-11 04:34:38,792] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-11 04:34:38,792] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default0]:[2022-09-11 04:34:38,891] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-11 04:34:38,891] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default4]:[2022-09-11 04:34:38,883] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-11 04:34:38,883] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default1]:[2022-09-11 04:34:39,053] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-11 04:34:39,053] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default6]:[2022-09-11 04:34:39,191] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-11 04:34:39,191] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default2]:[2022-09-11 04:34:39,245] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-11 04:34:39,245] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default3]:[2022-09-11 04:34:39,268] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-11 04:34:39,268] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default1]:[2022-09-11 04:34:39,378] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-11 04:34:39,378] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default5]:[2022-09-11 04:34:39,411] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-11 04:34:39,411] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default0]:[2022-09-11 04:34:39,368] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-11 04:34:39,368] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default0]:[2022-09-11 04:34:39,426] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-11 04:34:39,426] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default4]:[2022-09-11 04:34:39,462] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-11 04:34:39,462] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default3]:[2022-09-11 04:34:39,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-11 04:34:39,595] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default2]:[2022-09-11 04:34:39,543] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-11 04:34:39,543] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default1]:[2022-09-11 04:34:39,661] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-11 04:34:39,661] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default4]:[2022-09-11 04:34:39,701] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-11 04:34:39,701] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default4]:[2022-09-11 04:34:39,863] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-11 04:34:39,863] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default5]:[2022-09-11 04:34:39,917] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-11 04:34:39,917] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default2]:[2022-09-11 04:34:40,062] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-11 04:34:40,062] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default4]:[2022-09-11 04:34:40,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-11 04:34:40,221] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default7]:[2022-09-11 04:34:40,164] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-11 04:34:40,164] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default2]:[2022-09-11 04:34:40,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-11 04:34:40,314] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default7]:[2022-09-11 04:34:40,334] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-11 04:34:40,334] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default5]:[2022-09-11 04:34:40,381] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-11 04:34:40,381] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default7]:[2022-09-11 04:34:40,419] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-11 04:34:40,420] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default6]:[2022-09-11 04:34:40,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-11 04:34:40,823] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default6]:[2022-09-11 04:34:40,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-11 04:34:40,850] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default1]:[2022-09-11 04:34:40,864] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-11 04:34:40,864] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default6]:[2022-09-11 04:34:40,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-11 04:34:40,947] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default3]:[2022-09-11 04:34:40,988] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-11 04:34:40,988] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default0]:[2022-09-11 04:34:41,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-11 04:34:41,097] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default1]:[2022-09-11 04:34:41,116] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-11 04:34:41,116] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default2]:[2022-09-11 04:34:41,111] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-11 04:34:41,111] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default5]:[2022-09-11 04:34:41,220] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-11 04:34:41,220] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default5]:[2022-09-11 04:34:41,273] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-11 04:34:41,273] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default3]:[2022-09-11 04:34:41,426] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-11 04:34:41,427] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default0]:[2022-09-11 04:34:41,477] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-11 04:34:41,477] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default0]:[2022-09-11 04:34:41,481] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-11 04:34:41,482] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default1]:[2022-09-11 04:34:41,522] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-11 04:34:41,522] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default4]:[2022-09-11 04:34:41,503] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-11 04:34:41,503] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default0]:[2022-09-11 04:34:41,589] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-11 04:34:41,589] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default3]:[2022-09-11 04:34:41,640] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-11 04:34:41,640] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default2]:[2022-09-11 04:34:41,935] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-11 04:34:41,935] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default7]:[2022-09-11 04:34:42,026] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-11 04:34:42,026] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default2]:[2022-09-11 04:34:42,100] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-11 04:34:42,100] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default1]:[2022-09-11 04:34:42,031] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-11 04:34:42,031] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default4]:[2022-09-11 04:34:42,094] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-11 04:34:42,094] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default7]:[2022-09-11 04:34:42,258] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-11 04:34:42,258] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default6]:[2022-09-11 04:34:42,308] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-11 04:34:42,308] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default3]:[2022-09-11 04:34:42,436] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-11 04:34:42,436] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default6]:[2022-09-11 04:34:42,442] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-11 04:34:42,442] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default0]:[2022-09-11 04:34:42,521] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-11 04:34:42,523] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default1]:[2022-09-11 04:34:43,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-11 04:34:43,089] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default3]:[2022-09-11 04:34:43,610] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-11 04:34:43,610] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default2]:[2022-09-11 04:34:43,752] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-11 04:34:43,752] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default7]:[2022-09-11 04:34:44,457] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-11 04:34:44,458] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default4]:[2022-09-11 04:34:45,205] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-11 04:34:45,205] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default5]:[2022-09-11 04:34:45,762] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-11 04:34:45,762] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default6]:[2022-09-11 04:34:46,182] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-11 04:34:46,182] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default3]:[2022-09-11 04:34:48,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-11 04:34:48,595] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default0]:[2022-09-11 04:34:52,569] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-11 04:34:52,569] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default1]:[2022-09-11 04:34:52,844] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-11 04:34:52,844] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default2]:[2022-09-11 04:34:52,834] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-11 04:34:52,835] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default7]:[2022-09-11 04:34:55,419] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-11 04:34:55,419] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default4]:[2022-09-11 04:34:55,872] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-11 04:34:55,872] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default5]:[2022-09-11 04:34:55,966] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-11 04:34:55,966] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,145] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-11 04:34:56,145] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step1992/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]: successfully saved checkpoint at iteration 1992 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:time (ms) | save-checkpoint: 36407.46 -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default3]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default1]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default0]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default2]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default4]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default6]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default5]:[2022-09-11 04:34:56,146] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1992 is ready now! -[default7]: iteration 1993/ 3100 | consumed samples: 4081664 | consumed tokens: 8359247872 | elapsed time per iteration (s): 176.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.379546E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.595 | TFLOPs: 118.37 | -[default7]: iteration 1994/ 3100 | consumed samples: 4083712 | consumed tokens: 8363442176 | elapsed time per iteration (s): 141.86 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.527539E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 1995/ 3100 | consumed samples: 4085760 | consumed tokens: 8367636480 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.387226E-01 | grad norm: 0.564 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.79 | -[default7]: iteration 1996/ 3100 | consumed samples: 4087808 | consumed tokens: 8371830784 | elapsed time per iteration (s): 141.35 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.385984E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.489 | TFLOPs: 147.91 | -[default7]: iteration 1997/ 3100 | consumed samples: 4089856 | consumed tokens: 8376025088 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.491014E-01 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 1998/ 3100 | consumed samples: 4091904 | consumed tokens: 8380219392 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.486618E-01 | grad norm: 0.601 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 1999/ 3100 | consumed samples: 4093952 | consumed tokens: 8384413696 | elapsed time per iteration (s): 140.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.425252E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.572 | TFLOPs: 148.76 | -[default0]:[2022-09-11 04:53:45,932] [INFO] [logging.py:68:log_dist] [Rank 0] step=2000, skipped=0, lr=[2e-05, 2e-05], mom=[(0.9, 0.95), (0.9, 0.95)] -[default0]:steps: 2000 loss: 0.7448 iter time (s): 141.233 samples/sec: 14.501 -[default7]: iteration 2000/ 3100 | consumed samples: 4096000 | consumed tokens: 8388608000 | elapsed time per iteration (s): 141.05 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.447802E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.519 | TFLOPs: 148.22 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]:validation_pretraining loss at iteration 2000 | lm loss value: 2.425698E+00 | lm loss PPL: 1.131012E+01 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]: iteration 2001/ 3100 | consumed samples: 4098048 | consumed tokens: 8392802304 | elapsed time per iteration (s): 183.22 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.414933E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.178 | TFLOPs: 114.11 | -[default7]: iteration 2002/ 3100 | consumed samples: 4100096 | consumed tokens: 8396996608 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.472130E-01 | grad norm: 0.565 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 2003/ 3100 | consumed samples: 4102144 | consumed tokens: 8401190912 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.482744E-01 | grad norm: 0.627 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 2004/ 3100 | consumed samples: 4104192 | consumed tokens: 8405385216 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.547663E-01 | grad norm: 0.863 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 2005/ 3100 | consumed samples: 4106240 | consumed tokens: 8409579520 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.576518E-01 | grad norm: 4.824 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.76 | -[default7]: iteration 2006/ 3100 | consumed samples: 4108288 | consumed tokens: 8413773824 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.544819E-01 | grad norm: 1.711 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.54 | -[default7]: iteration 2007/ 3100 | consumed samples: 4110336 | consumed tokens: 8417968128 | elapsed time per iteration (s): 141.92 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.572191E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.431 | TFLOPs: 147.32 | -[default7]: iteration 2008/ 3100 | consumed samples: 4112384 | consumed tokens: 8422162432 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.639377E-01 | grad norm: 0.431 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 2009/ 3100 | consumed samples: 4114432 | consumed tokens: 8426356736 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.398143E-01 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 2010/ 3100 | consumed samples: 4116480 | consumed tokens: 8430551040 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.471830E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 2011/ 3100 | consumed samples: 4118528 | consumed tokens: 8434745344 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.587160E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 2012/ 3100 | consumed samples: 4120576 | consumed tokens: 8438939648 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.428085E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 2013/ 3100 | consumed samples: 4122624 | consumed tokens: 8443133952 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.220705E-01 | grad norm: 0.438 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 2014/ 3100 | consumed samples: 4124672 | consumed tokens: 8447328256 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.476459E-01 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 2015/ 3100 | consumed samples: 4126720 | consumed tokens: 8451522560 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.634326E-01 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 2016/ 3100 | consumed samples: 4128768 | consumed tokens: 8455716864 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.475870E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 2017/ 3100 | consumed samples: 4130816 | consumed tokens: 8459911168 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.402470E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 2018/ 3100 | consumed samples: 4132864 | consumed tokens: 8464105472 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.451587E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 2019/ 3100 | consumed samples: 4134912 | consumed tokens: 8468299776 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.366836E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.61 | -[default7]: iteration 2020/ 3100 | consumed samples: 4136960 | consumed tokens: 8472494080 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.391903E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 2021/ 3100 | consumed samples: 4139008 | consumed tokens: 8476688384 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.537246E-01 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 2022/ 3100 | consumed samples: 4141056 | consumed tokens: 8480882688 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.450636E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.75 | -[default7]: iteration 2023/ 3100 | consumed samples: 4143104 | consumed tokens: 8485076992 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.479554E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 2024/ 3100 | consumed samples: 4145152 | consumed tokens: 8489271296 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.512291E-01 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 2025/ 3100 | consumed samples: 4147200 | consumed tokens: 8493465600 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.389866E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.62 | -[default7]: iteration 2026/ 3100 | consumed samples: 4149248 | consumed tokens: 8497659904 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.395818E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 2027/ 3100 | consumed samples: 4151296 | consumed tokens: 8501854208 | elapsed time per iteration (s): 141.14 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.444822E-01 | grad norm: 0.401 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.510 | TFLOPs: 148.13 | -[default7]: iteration 2028/ 3100 | consumed samples: 4153344 | consumed tokens: 8506048512 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.466800E-01 | grad norm: 3.456 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 2029/ 3100 | consumed samples: 4155392 | consumed tokens: 8510242816 | elapsed time per iteration (s): 140.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.380302E-01 | grad norm: 0.435 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.530 | TFLOPs: 148.33 | -[default7]: iteration 2030/ 3100 | consumed samples: 4157440 | consumed tokens: 8514437120 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.521067E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 2031/ 3100 | consumed samples: 4159488 | consumed tokens: 8518631424 | elapsed time per iteration (s): 140.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.420563E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.590 | TFLOPs: 148.94 | -[default7]: iteration 2032/ 3100 | consumed samples: 4161536 | consumed tokens: 8522825728 | elapsed time per iteration (s): 140.38 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.287502E-01 | grad norm: 0.406 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.589 | TFLOPs: 148.93 | -[default7]: iteration 2033/ 3100 | consumed samples: 4163584 | consumed tokens: 8527020032 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.449396E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 2034/ 3100 | consumed samples: 4165632 | consumed tokens: 8531214336 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.389866E-01 | grad norm: 0.397 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 2035/ 3100 | consumed samples: 4167680 | consumed tokens: 8535408640 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.414156E-01 | grad norm: 0.350 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 2036/ 3100 | consumed samples: 4169728 | consumed tokens: 8539602944 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.460349E-01 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 2037/ 3100 | consumed samples: 4171776 | consumed tokens: 8543797248 | elapsed time per iteration (s): 140.22 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.513320E-01 | grad norm: 0.374 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.606 | TFLOPs: 149.10 | -[default7]: iteration 2038/ 3100 | consumed samples: 4173824 | consumed tokens: 8547991552 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.344204E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 2039/ 3100 | consumed samples: 4175872 | consumed tokens: 8552185856 | elapsed time per iteration (s): 140.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.477951E-01 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.570 | TFLOPs: 148.74 | -[default7]: iteration 2040/ 3100 | consumed samples: 4177920 | consumed tokens: 8556380160 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.478163E-01 | grad norm: 0.347 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.481 | TFLOPs: 147.83 | -[default7]: iteration 2041/ 3100 | consumed samples: 4179968 | consumed tokens: 8560574464 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.346773E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 2042/ 3100 | consumed samples: 4182016 | consumed tokens: 8564768768 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.414088E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 2043/ 3100 | consumed samples: 4184064 | consumed tokens: 8568963072 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.499596E-01 | grad norm: 1.204 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 2044/ 3100 | consumed samples: 4186112 | consumed tokens: 8573157376 | elapsed time per iteration (s): 141.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.411125E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.484 | TFLOPs: 147.86 | -[default7]: iteration 2045/ 3100 | consumed samples: 4188160 | consumed tokens: 8577351680 | elapsed time per iteration (s): 141.32 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.449667E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.492 | TFLOPs: 147.94 | -[default7]: iteration 2046/ 3100 | consumed samples: 4190208 | consumed tokens: 8581545984 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.290689E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 2047/ 3100 | consumed samples: 4192256 | consumed tokens: 8585740288 | elapsed time per iteration (s): 140.19 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.378956E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.609 | TFLOPs: 149.14 | -[default7]: iteration 2048/ 3100 | consumed samples: 4194304 | consumed tokens: 8589934592 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.258061E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 2049/ 3100 | consumed samples: 4196352 | consumed tokens: 8594128896 | elapsed time per iteration (s): 140.23 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.460822E-01 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.604 | TFLOPs: 149.09 | -[default7]: iteration 2050/ 3100 | consumed samples: 4198400 | consumed tokens: 8598323200 | elapsed time per iteration (s): 140.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.464762E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.544 | TFLOPs: 148.48 | -[default7]: iteration 2051/ 3100 | consumed samples: 4200448 | consumed tokens: 8602517504 | elapsed time per iteration (s): 141.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.525773E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.487 | TFLOPs: 147.89 | -[default7]: iteration 2052/ 3100 | consumed samples: 4202496 | consumed tokens: 8606711808 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.469450E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.499 | TFLOPs: 148.01 | -[default7]: iteration 2053/ 3100 | consumed samples: 4204544 | consumed tokens: 8610906112 | elapsed time per iteration (s): 140.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.296010E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.566 | TFLOPs: 148.70 | -[default7]: iteration 2054/ 3100 | consumed samples: 4206592 | consumed tokens: 8615100416 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.357377E-01 | grad norm: 0.400 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 2055/ 3100 | consumed samples: 4208640 | consumed tokens: 8619294720 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.229784E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.78 | -[default7]: iteration 2056/ 3100 | consumed samples: 4210688 | consumed tokens: 8623489024 | elapsed time per iteration (s): 141.27 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.435234E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.497 | TFLOPs: 147.99 | -[default7]: iteration 2057/ 3100 | consumed samples: 4212736 | consumed tokens: 8627683328 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.529157E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 2058/ 3100 | consumed samples: 4214784 | consumed tokens: 8631877632 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.460441E-01 | grad norm: 0.350 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.74 | -[default7]: iteration 2059/ 3100 | consumed samples: 4216832 | consumed tokens: 8636071936 | elapsed time per iteration (s): 141.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.306656E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 2060/ 3100 | consumed samples: 4218880 | consumed tokens: 8640266240 | elapsed time per iteration (s): 141.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.306832E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.82 | -[default7]: iteration 2061/ 3100 | consumed samples: 4220928 | consumed tokens: 8644460544 | elapsed time per iteration (s): 141.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.424043E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.490 | TFLOPs: 147.93 | -[default7]: iteration 2062/ 3100 | consumed samples: 4222976 | consumed tokens: 8648654848 | elapsed time per iteration (s): 140.09 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.356107E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.620 | TFLOPs: 149.24 | -[default7]: iteration 2063/ 3100 | consumed samples: 4225024 | consumed tokens: 8652849152 | elapsed time per iteration (s): 139.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.457401E-01 | grad norm: 0.352 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.657 | TFLOPs: 149.63 | -[default7]: iteration 2064/ 3100 | consumed samples: 4227072 | consumed tokens: 8657043456 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.347519E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 2065/ 3100 | consumed samples: 4229120 | consumed tokens: 8661237760 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.381247E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 2066/ 3100 | consumed samples: 4231168 | consumed tokens: 8665432064 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.349671E-01 | grad norm: 0.424 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 2067/ 3100 | consumed samples: 4233216 | consumed tokens: 8669626368 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.409582E-01 | grad norm: 0.443 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.74 | -[default7]: iteration 2068/ 3100 | consumed samples: 4235264 | consumed tokens: 8673820672 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.488043E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 2069/ 3100 | consumed samples: 4237312 | consumed tokens: 8678014976 | elapsed time per iteration (s): 141.21 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.452810E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.503 | TFLOPs: 148.05 | -[default7]: iteration 2070/ 3100 | consumed samples: 4239360 | consumed tokens: 8682209280 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.376490E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 2071/ 3100 | consumed samples: 4241408 | consumed tokens: 8686403584 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.390484E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.476 | TFLOPs: 147.78 | -[default7]: iteration 2072/ 3100 | consumed samples: 4243456 | consumed tokens: 8690597888 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.385734E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 2073/ 3100 | consumed samples: 4245504 | consumed tokens: 8694792192 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.481527E-01 | grad norm: 0.336 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 2074/ 3100 | consumed samples: 4247552 | consumed tokens: 8698986496 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.316664E-01 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 2075/ 3100 | consumed samples: 4249600 | consumed tokens: 8703180800 | elapsed time per iteration (s): 140.05 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.423118E-01 | grad norm: 0.350 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.624 | TFLOPs: 149.28 | -[default7]: iteration 2076/ 3100 | consumed samples: 4251648 | consumed tokens: 8707375104 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.345178E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 2077/ 3100 | consumed samples: 4253696 | consumed tokens: 8711569408 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.422757E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.70 | -[default7]: iteration 2078/ 3100 | consumed samples: 4255744 | consumed tokens: 8715763712 | elapsed time per iteration (s): 140.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.329479E-01 | grad norm: 0.476 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.554 | TFLOPs: 148.57 | -[default7]: iteration 2079/ 3100 | consumed samples: 4257792 | consumed tokens: 8719958016 | elapsed time per iteration (s): 140.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.446127E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.581 | TFLOPs: 148.85 | -[default7]: iteration 2080/ 3100 | consumed samples: 4259840 | consumed tokens: 8724152320 | elapsed time per iteration (s): 141.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.386400E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.487 | TFLOPs: 147.89 | -[default7]: iteration 2081/ 3100 | consumed samples: 4261888 | consumed tokens: 8728346624 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.324505E-01 | grad norm: 0.335 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 2082/ 3100 | consumed samples: 4263936 | consumed tokens: 8732540928 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.411726E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 2083/ 3100 | consumed samples: 4265984 | consumed tokens: 8736735232 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.283276E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 2084/ 3100 | consumed samples: 4268032 | consumed tokens: 8740929536 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.509000E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 2085/ 3100 | consumed samples: 4270080 | consumed tokens: 8745123840 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.193062E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 2086/ 3100 | consumed samples: 4272128 | consumed tokens: 8749318144 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.277573E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 2087/ 3100 | consumed samples: 4274176 | consumed tokens: 8753512448 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.543262E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 2088/ 3100 | consumed samples: 4276224 | consumed tokens: 8757706752 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.207434E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 2089/ 3100 | consumed samples: 4278272 | consumed tokens: 8761901056 | elapsed time per iteration (s): 140.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.445006E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.582 | TFLOPs: 148.86 | -[default7]: iteration 2090/ 3100 | consumed samples: 4280320 | consumed tokens: 8766095360 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.255908E-01 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 2091/ 3100 | consumed samples: 4282368 | consumed tokens: 8770289664 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.421231E-01 | grad norm: 0.341 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 2092/ 3100 | consumed samples: 4284416 | consumed tokens: 8774483968 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.407969E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 2093/ 3100 | consumed samples: 4286464 | consumed tokens: 8778678272 | elapsed time per iteration (s): 141.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.236419E-01 | grad norm: 3.620 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.525 | TFLOPs: 148.28 | -[default7]: iteration 2094/ 3100 | consumed samples: 4288512 | consumed tokens: 8782872576 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.354796E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 2095/ 3100 | consumed samples: 4290560 | consumed tokens: 8787066880 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.363845E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 2096/ 3100 | consumed samples: 4292608 | consumed tokens: 8791261184 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.404681E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 2097/ 3100 | consumed samples: 4294656 | consumed tokens: 8795455488 | elapsed time per iteration (s): 140.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.274284E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.544 | TFLOPs: 148.47 | -[default7]: iteration 2098/ 3100 | consumed samples: 4296704 | consumed tokens: 8799649792 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.405519E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.74 | -[default7]: iteration 2099/ 3100 | consumed samples: 4298752 | consumed tokens: 8803844096 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.273945E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 2100/ 3100 | consumed samples: 4300800 | consumed tokens: 8808038400 | elapsed time per iteration (s): 141.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.406253E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 2101/ 3100 | consumed samples: 4302848 | consumed tokens: 8812232704 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.377449E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 2102/ 3100 | consumed samples: 4304896 | consumed tokens: 8816427008 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.565918E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 2103/ 3100 | consumed samples: 4306944 | consumed tokens: 8820621312 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.401937E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 2104/ 3100 | consumed samples: 4308992 | consumed tokens: 8824815616 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.467316E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 2105/ 3100 | consumed samples: 4311040 | consumed tokens: 8829009920 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.440394E-01 | grad norm: 1.286 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 2106/ 3100 | consumed samples: 4313088 | consumed tokens: 8833204224 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.346557E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 2107/ 3100 | consumed samples: 4315136 | consumed tokens: 8837398528 | elapsed time per iteration (s): 141.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.351410E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.482 | TFLOPs: 147.84 | -[default7]: iteration 2108/ 3100 | consumed samples: 4317184 | consumed tokens: 8841592832 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.162656E-01 | grad norm: 0.996 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 2109/ 3100 | consumed samples: 4319232 | consumed tokens: 8845787136 | elapsed time per iteration (s): 141.29 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.391568E-01 | grad norm: 0.337 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.495 | TFLOPs: 147.97 | -[default7]: iteration 2110/ 3100 | consumed samples: 4321280 | consumed tokens: 8849981440 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.336820E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.80 | -[default7]: iteration 2111/ 3100 | consumed samples: 4323328 | consumed tokens: 8854175744 | elapsed time per iteration (s): 141.39 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.357779E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.485 | TFLOPs: 147.87 | -[default7]: iteration 2112/ 3100 | consumed samples: 4325376 | consumed tokens: 8858370048 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.310466E-01 | grad norm: 0.342 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 2113/ 3100 | consumed samples: 4327424 | consumed tokens: 8862564352 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.206885E-01 | grad norm: 0.341 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.64 | -[default7]: iteration 2114/ 3100 | consumed samples: 4329472 | consumed tokens: 8866758656 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.258615E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.79 | -[default7]: iteration 2115/ 3100 | consumed samples: 4331520 | consumed tokens: 8870952960 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.438490E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 2116/ 3100 | consumed samples: 4333568 | consumed tokens: 8875147264 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.292660E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 2117/ 3100 | consumed samples: 4335616 | consumed tokens: 8879341568 | elapsed time per iteration (s): 141.26 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.456682E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.498 | TFLOPs: 148.00 | -[default7]: iteration 2118/ 3100 | consumed samples: 4337664 | consumed tokens: 8883535872 | elapsed time per iteration (s): 141.38 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.283499E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.486 | TFLOPs: 147.87 | -[default7]: iteration 2119/ 3100 | consumed samples: 4339712 | consumed tokens: 8887730176 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.326220E-01 | grad norm: 0.347 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 2120/ 3100 | consumed samples: 4341760 | consumed tokens: 8891924480 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.303486E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 2121/ 3100 | consumed samples: 4343808 | consumed tokens: 8896118784 | elapsed time per iteration (s): 141.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.410229E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.439 | TFLOPs: 147.40 | -[default7]: iteration 2122/ 3100 | consumed samples: 4345856 | consumed tokens: 8900313088 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.403882E-01 | grad norm: 0.404 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 2123/ 3100 | consumed samples: 4347904 | consumed tokens: 8904507392 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.229683E-01 | grad norm: 0.334 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 2124/ 3100 | consumed samples: 4349952 | consumed tokens: 8908701696 | elapsed time per iteration (s): 141.04 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.370159E-01 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.520 | TFLOPs: 148.23 | -[default7]: iteration 2125/ 3100 | consumed samples: 4352000 | consumed tokens: 8912896000 | elapsed time per iteration (s): 141.09 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.463923E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.516 | TFLOPs: 148.19 | -[default7]: iteration 2126/ 3100 | consumed samples: 4354048 | consumed tokens: 8917090304 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.384860E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 2127/ 3100 | consumed samples: 4356096 | consumed tokens: 8921284608 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.234438E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 2128/ 3100 | consumed samples: 4358144 | consumed tokens: 8925478912 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.410901E-01 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 2129/ 3100 | consumed samples: 4360192 | consumed tokens: 8929673216 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.247515E-01 | grad norm: 0.365 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 2130/ 3100 | consumed samples: 4362240 | consumed tokens: 8933867520 | elapsed time per iteration (s): 140.23 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.235085E-01 | grad norm: 0.358 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.605 | TFLOPs: 149.09 | -[default7]: iteration 2131/ 3100 | consumed samples: 4364288 | consumed tokens: 8938061824 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.348924E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 2132/ 3100 | consumed samples: 4366336 | consumed tokens: 8942256128 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.343862E-01 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 2133/ 3100 | consumed samples: 4368384 | consumed tokens: 8946450432 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.323183E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 2134/ 3100 | consumed samples: 4370432 | consumed tokens: 8950644736 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.390875E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 2135/ 3100 | consumed samples: 4372480 | consumed tokens: 8954839040 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.356781E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.66 | -[default7]: iteration 2136/ 3100 | consumed samples: 4374528 | consumed tokens: 8959033344 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.447636E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 2137/ 3100 | consumed samples: 4376576 | consumed tokens: 8963227648 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.310953E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 2138/ 3100 | consumed samples: 4378624 | consumed tokens: 8967421952 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.319149E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 2139/ 3100 | consumed samples: 4380672 | consumed tokens: 8971616256 | elapsed time per iteration (s): 141.11 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.263819E-01 | grad norm: 0.335 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.514 | TFLOPs: 148.16 | -[default7]: iteration 2140/ 3100 | consumed samples: 4382720 | consumed tokens: 8975810560 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.389215E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 2141/ 3100 | consumed samples: 4384768 | consumed tokens: 8980004864 | elapsed time per iteration (s): 140.27 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.392001E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.600 | TFLOPs: 149.05 | -[default7]: iteration 2142/ 3100 | consumed samples: 4386816 | consumed tokens: 8984199168 | elapsed time per iteration (s): 140.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.238272E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.560 | TFLOPs: 148.63 | -[default7]: iteration 2143/ 3100 | consumed samples: 4388864 | consumed tokens: 8988393472 | elapsed time per iteration (s): 141.43 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.162344E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.480 | TFLOPs: 147.82 | -[default7]: iteration 2144/ 3100 | consumed samples: 4390912 | consumed tokens: 8992587776 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.254220E-01 | grad norm: 0.362 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 2145/ 3100 | consumed samples: 4392960 | consumed tokens: 8996782080 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.335848E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 2146/ 3100 | consumed samples: 4395008 | consumed tokens: 9000976384 | elapsed time per iteration (s): 141.41 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.279033E-01 | grad norm: 0.409 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.483 | TFLOPs: 147.85 | -[default7]: iteration 2147/ 3100 | consumed samples: 4397056 | consumed tokens: 9005170688 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.203154E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 2148/ 3100 | consumed samples: 4399104 | consumed tokens: 9009364992 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.302870E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 2149/ 3100 | consumed samples: 4401152 | consumed tokens: 9013559296 | elapsed time per iteration (s): 141.02 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.264012E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.523 | TFLOPs: 148.26 | -[default7]: iteration 2150/ 3100 | consumed samples: 4403200 | consumed tokens: 9017753600 | elapsed time per iteration (s): 141.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.298160E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.525 | TFLOPs: 148.28 | -[default7]: iteration 2151/ 3100 | consumed samples: 4405248 | consumed tokens: 9021947904 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.329667E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 2152/ 3100 | consumed samples: 4407296 | consumed tokens: 9026142208 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.294875E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 2153/ 3100 | consumed samples: 4409344 | consumed tokens: 9030336512 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.269253E-01 | grad norm: 0.342 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 2154/ 3100 | consumed samples: 4411392 | consumed tokens: 9034530816 | elapsed time per iteration (s): 142.09 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.392634E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.413 | TFLOPs: 147.14 | -[default7]: iteration 2155/ 3100 | consumed samples: 4413440 | consumed tokens: 9038725120 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.241573E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.51 | -[default7]: iteration 2156/ 3100 | consumed samples: 4415488 | consumed tokens: 9042919424 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.325580E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 | -[default7]: iteration 2157/ 3100 | consumed samples: 4417536 | consumed tokens: 9047113728 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.152631E-01 | grad norm: 0.341 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 2158/ 3100 | consumed samples: 4419584 | consumed tokens: 9051308032 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.409434E-01 | grad norm: 0.336 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 2159/ 3100 | consumed samples: 4421632 | consumed tokens: 9055502336 | elapsed time per iteration (s): 140.18 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.358358E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.610 | TFLOPs: 149.15 | -[default7]: iteration 2160/ 3100 | consumed samples: 4423680 | consumed tokens: 9059696640 | elapsed time per iteration (s): 141.20 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.184622E-01 | grad norm: 0.331 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.505 | TFLOPs: 148.07 | -[default7]: iteration 2161/ 3100 | consumed samples: 4425728 | consumed tokens: 9063890944 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.384659E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 2162/ 3100 | consumed samples: 4427776 | consumed tokens: 9068085248 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.402006E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 2163/ 3100 | consumed samples: 4429824 | consumed tokens: 9072279552 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.201594E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 2164/ 3100 | consumed samples: 4431872 | consumed tokens: 9076473856 | elapsed time per iteration (s): 141.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.421973E-01 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.499 | TFLOPs: 148.01 | -[default7]: iteration 2165/ 3100 | consumed samples: 4433920 | consumed tokens: 9080668160 | elapsed time per iteration (s): 141.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.411165E-01 | grad norm: 0.428 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.487 | TFLOPs: 147.89 | -[default7]: iteration 2166/ 3100 | consumed samples: 4435968 | consumed tokens: 9084862464 | elapsed time per iteration (s): 140.26 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.421256E-01 | grad norm: 0.347 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.601 | TFLOPs: 149.05 | -[default7]: iteration 2167/ 3100 | consumed samples: 4438016 | consumed tokens: 9089056768 | elapsed time per iteration (s): 141.09 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.323651E-01 | grad norm: 0.347 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.516 | TFLOPs: 148.19 | -[default7]: iteration 2168/ 3100 | consumed samples: 4440064 | consumed tokens: 9093251072 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.387488E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 2169/ 3100 | consumed samples: 4442112 | consumed tokens: 9097445376 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.277624E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 2170/ 3100 | consumed samples: 4444160 | consumed tokens: 9101639680 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.106416E-01 | grad norm: 0.347 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.80 | -[default7]: iteration 2171/ 3100 | consumed samples: 4446208 | consumed tokens: 9105833984 | elapsed time per iteration (s): 141.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.274037E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.484 | TFLOPs: 147.86 | -[default7]: iteration 2172/ 3100 | consumed samples: 4448256 | consumed tokens: 9110028288 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.327082E-01 | grad norm: 0.352 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 2173/ 3100 | consumed samples: 4450304 | consumed tokens: 9114222592 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.189927E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 2174/ 3100 | consumed samples: 4452352 | consumed tokens: 9118416896 | elapsed time per iteration (s): 141.47 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.354578E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.477 | TFLOPs: 147.78 | -[default7]: iteration 2175/ 3100 | consumed samples: 4454400 | consumed tokens: 9122611200 | elapsed time per iteration (s): 140.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.399134E-01 | grad norm: 0.335 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.587 | TFLOPs: 148.91 | -[default7]: iteration 2176/ 3100 | consumed samples: 4456448 | consumed tokens: 9126805504 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.390726E-01 | grad norm: 0.338 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 2177/ 3100 | consumed samples: 4458496 | consumed tokens: 9130999808 | elapsed time per iteration (s): 141.08 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.257507E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.516 | TFLOPs: 148.19 | -[default7]: iteration 2178/ 3100 | consumed samples: 4460544 | consumed tokens: 9135194112 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.349805E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 2179/ 3100 | consumed samples: 4462592 | consumed tokens: 9139388416 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.419482E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 2180/ 3100 | consumed samples: 4464640 | consumed tokens: 9143582720 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.196565E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 2181/ 3100 | consumed samples: 4466688 | consumed tokens: 9147777024 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.199653E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.47 | -[default7]: iteration 2182/ 3100 | consumed samples: 4468736 | consumed tokens: 9151971328 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.146042E-01 | grad norm: 0.358 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 2183/ 3100 | consumed samples: 4470784 | consumed tokens: 9156165632 | elapsed time per iteration (s): 142.99 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.310041E-01 | grad norm: 0.336 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.323 | TFLOPs: 146.21 | -[default7]: iteration 2184/ 3100 | consumed samples: 4472832 | consumed tokens: 9160359936 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.365180E-01 | grad norm: 0.350 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.472 | TFLOPs: 147.73 | -[default7]: iteration 2185/ 3100 | consumed samples: 4474880 | consumed tokens: 9164554240 | elapsed time per iteration (s): 141.16 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.479917E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.509 | TFLOPs: 148.11 | -[default7]: iteration 2186/ 3100 | consumed samples: 4476928 | consumed tokens: 9168748544 | elapsed time per iteration (s): 142.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.214113E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.382 | TFLOPs: 146.81 | -[default7]: iteration 2187/ 3100 | consumed samples: 4478976 | consumed tokens: 9172942848 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.220213E-01 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 2188/ 3100 | consumed samples: 4481024 | consumed tokens: 9177137152 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.046787E-01 | grad norm: 0.376 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 2189/ 3100 | consumed samples: 4483072 | consumed tokens: 9181331456 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.286308E-01 | grad norm: 0.462 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 2190/ 3100 | consumed samples: 4485120 | consumed tokens: 9185525760 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.178335E-01 | grad norm: 0.399 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 2191/ 3100 | consumed samples: 4487168 | consumed tokens: 9189720064 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.146949E-01 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 2192/ 3100 | consumed samples: 4489216 | consumed tokens: 9193914368 | elapsed time per iteration (s): 140.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.200203E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.561 | TFLOPs: 148.65 | -[default7]: iteration 2193/ 3100 | consumed samples: 4491264 | consumed tokens: 9198108672 | elapsed time per iteration (s): 141.46 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.450780E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.478 | TFLOPs: 147.79 | -[default7]: iteration 2194/ 3100 | consumed samples: 4493312 | consumed tokens: 9202302976 | elapsed time per iteration (s): 140.36 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.288993E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.591 | TFLOPs: 148.95 | -[default7]: iteration 2195/ 3100 | consumed samples: 4495360 | consumed tokens: 9206497280 | elapsed time per iteration (s): 139.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.241597E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.639 | TFLOPs: 149.44 | -[default7]: iteration 2196/ 3100 | consumed samples: 4497408 | consumed tokens: 9210691584 | elapsed time per iteration (s): 140.94 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.508590E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.531 | TFLOPs: 148.34 | -[default7]: iteration 2197/ 3100 | consumed samples: 4499456 | consumed tokens: 9214885888 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.329370E-01 | grad norm: 0.342 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 2198/ 3100 | consumed samples: 4501504 | consumed tokens: 9219080192 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.132103E-01 | grad norm: 0.340 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 2199/ 3100 | consumed samples: 4503552 | consumed tokens: 9223274496 | elapsed time per iteration (s): 140.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.375839E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.546 | TFLOPs: 148.50 | -[default7]: iteration 2200/ 3100 | consumed samples: 4505600 | consumed tokens: 9227468800 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.017889E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.76 | -[default7]: iteration 2201/ 3100 | consumed samples: 4507648 | consumed tokens: 9231663104 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.297584E-01 | grad norm: 0.396 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.54 | -[default7]: iteration 2202/ 3100 | consumed samples: 4509696 | consumed tokens: 9235857408 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.263119E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 2203/ 3100 | consumed samples: 4511744 | consumed tokens: 9240051712 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.084410E-01 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 2204/ 3100 | consumed samples: 4513792 | consumed tokens: 9244246016 | elapsed time per iteration (s): 141.04 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.341641E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.520 | TFLOPs: 148.23 | -[default7]: iteration 2205/ 3100 | consumed samples: 4515840 | consumed tokens: 9248440320 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.253324E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.52 | -[default7]: iteration 2206/ 3100 | consumed samples: 4517888 | consumed tokens: 9252634624 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.247760E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 2207/ 3100 | consumed samples: 4519936 | consumed tokens: 9256828928 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.111655E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 2208/ 3100 | consumed samples: 4521984 | consumed tokens: 9261023232 | elapsed time per iteration (s): 140.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.258772E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.573 | TFLOPs: 148.77 | -[default7]: iteration 2209/ 3100 | consumed samples: 4524032 | consumed tokens: 9265217536 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.201117E-01 | grad norm: 0.358 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.63 | -[default7]: iteration 2210/ 3100 | consumed samples: 4526080 | consumed tokens: 9269411840 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.326931E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 2211/ 3100 | consumed samples: 4528128 | consumed tokens: 9273606144 | elapsed time per iteration (s): 140.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.093195E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.568 | TFLOPs: 148.71 | -[default7]: iteration 2212/ 3100 | consumed samples: 4530176 | consumed tokens: 9277800448 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.278312E-01 | grad norm: 0.387 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 2213/ 3100 | consumed samples: 4532224 | consumed tokens: 9281994752 | elapsed time per iteration (s): 139.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.233409E-01 | grad norm: 0.347 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.647 | TFLOPs: 149.53 | -[default7]: iteration 2214/ 3100 | consumed samples: 4534272 | consumed tokens: 9286189056 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.131695E-01 | grad norm: 0.350 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 2215/ 3100 | consumed samples: 4536320 | consumed tokens: 9290383360 | elapsed time per iteration (s): 141.91 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.180855E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.432 | TFLOPs: 147.33 | -[default7]: iteration 2216/ 3100 | consumed samples: 4538368 | consumed tokens: 9294577664 | elapsed time per iteration (s): 141.52 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.162966E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 2217/ 3100 | consumed samples: 4540416 | consumed tokens: 9298771968 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.297294E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.63 | -[default7]: iteration 2218/ 3100 | consumed samples: 4542464 | consumed tokens: 9302966272 | elapsed time per iteration (s): 141.00 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.299551E-01 | grad norm: 0.338 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.525 | TFLOPs: 148.28 | -[default7]: iteration 2219/ 3100 | consumed samples: 4544512 | consumed tokens: 9307160576 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.162129E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 2220/ 3100 | consumed samples: 4546560 | consumed tokens: 9311354880 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.151113E-01 | grad norm: 0.353 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.71 | -[default7]: iteration 2221/ 3100 | consumed samples: 4548608 | consumed tokens: 9315549184 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.156096E-01 | grad norm: 0.331 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.47 | -[default7]: iteration 2222/ 3100 | consumed samples: 4550656 | consumed tokens: 9319743488 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.359034E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 2223/ 3100 | consumed samples: 4552704 | consumed tokens: 9323937792 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.200698E-01 | grad norm: 0.337 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 2224/ 3100 | consumed samples: 4554752 | consumed tokens: 9328132096 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.219930E-01 | grad norm: 0.337 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 2225/ 3100 | consumed samples: 4556800 | consumed tokens: 9332326400 | elapsed time per iteration (s): 140.88 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.244580E-01 | grad norm: 0.385 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.538 | TFLOPs: 148.41 | -[default7]: iteration 2226/ 3100 | consumed samples: 4558848 | consumed tokens: 9336520704 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.260579E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.69 | -[default7]: iteration 2227/ 3100 | consumed samples: 4560896 | consumed tokens: 9340715008 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.218261E-01 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 2228/ 3100 | consumed samples: 4562944 | consumed tokens: 9344909312 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.289232E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 2229/ 3100 | consumed samples: 4564992 | consumed tokens: 9349103616 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.081753E-01 | grad norm: 0.382 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 2230/ 3100 | consumed samples: 4567040 | consumed tokens: 9353297920 | elapsed time per iteration (s): 140.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.243201E-01 | grad norm: 0.411 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.536 | TFLOPs: 148.39 | -[default7]: iteration 2231/ 3100 | consumed samples: 4569088 | consumed tokens: 9357492224 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.195415E-01 | grad norm: 0.444 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 2232/ 3100 | consumed samples: 4571136 | consumed tokens: 9361686528 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.175968E-01 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 2233/ 3100 | consumed samples: 4573184 | consumed tokens: 9365880832 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.310300E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.60 | -[default7]: iteration 2234/ 3100 | consumed samples: 4575232 | consumed tokens: 9370075136 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.201875E-01 | grad norm: 0.418 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 2235/ 3100 | consumed samples: 4577280 | consumed tokens: 9374269440 | elapsed time per iteration (s): 140.16 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.265930E-01 | grad norm: 0.352 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.612 | TFLOPs: 149.16 | -[default7]: iteration 2236/ 3100 | consumed samples: 4579328 | consumed tokens: 9378463744 | elapsed time per iteration (s): 141.34 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.155763E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.489 | TFLOPs: 147.91 | -[default7]: iteration 2237/ 3100 | consumed samples: 4581376 | consumed tokens: 9382658048 | elapsed time per iteration (s): 140.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.148261E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.544 | TFLOPs: 148.47 | -[default7]: iteration 2238/ 3100 | consumed samples: 4583424 | consumed tokens: 9386852352 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.200685E-01 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 2239/ 3100 | consumed samples: 4585472 | consumed tokens: 9391046656 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.311441E-01 | grad norm: 0.373 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 2240/ 3100 | consumed samples: 4587520 | consumed tokens: 9395240960 | elapsed time per iteration (s): 140.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.173488E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.555 | TFLOPs: 148.58 | -[default7]: iteration 2241/ 3100 | consumed samples: 4589568 | consumed tokens: 9399435264 | elapsed time per iteration (s): 140.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.188435E-01 | grad norm: 0.350 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.584 | TFLOPs: 148.88 | -[default4]:[2022-09-11 14:22:22,775] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_35-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,781] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_07-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,793] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_23-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,775] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_34-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,793] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_22-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,831] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_15-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,781] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_06-model_00-model_states.pt... -[default0]:saving checkpoint at iteration 2241 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-11 14:22:22,757] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step2241 is begin to save! -[default0]:[2022-09-11 14:22:22,858] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_12-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_31-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_16-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,868] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_62-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,868] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_29-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,868] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_28-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,868] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_50-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,862] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_70-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_52-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,877] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_24-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,857] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_13-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_61-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_54-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,897] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_71_model_states.pt... -[default0]:[2022-09-11 14:22:22,877] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_08-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,868] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_63-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_60-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,877] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_40-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_57-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,831] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_14-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_30-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_44-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,862] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_71-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,913] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_59-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_43-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,877] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_19-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,934] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_32-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_39-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_26-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,919] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_68-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,877] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_20-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,934] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_36-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,934] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_01-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,877] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_18-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_56-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,934] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_49-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_66-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,877] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_21-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_38-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_45-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_17-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_55-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,868] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_51-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_27-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_53-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_10-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,934] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_37-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_69-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_25-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_09-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,918] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_05-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,951] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_64-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,931] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_03-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,934] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_48-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_41-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_67-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,957] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_47-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,917] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_42-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,904] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_72-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,905] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_11-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,934] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_33-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,954] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_71_model_states.pt. -[default0]:[2022-09-11 14:22:22,913] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_58-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:22,951] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_65-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,957] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_46-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:22,954] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_04-model_00-model_states.pt... -[default4]:[2022-09-11 14:22:26,217] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_05-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,217] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_03_model_states.pt... -[default4]:[2022-09-11 14:22:26,219] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_03_model_states.pt. -[default0]:[2022-09-11 14:22:26,269] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_72-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,269] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_74-model_00-model_states.pt... -[default0]:[2022-09-11 14:22:26,273] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_74-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,273] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_70_model_states.pt... -[default0]:[2022-09-11 14:22:26,294] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_24-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_22_model_states.pt... -[default0]:[2022-09-11 14:22:26,297] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_22_model_states.pt. -[default0]:[2022-09-11 14:22:26,275] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_70_model_states.pt. -[default4]:[2022-09-11 14:22:26,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_39-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,335] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_37_model_states.pt... -[default4]:[2022-09-11 14:22:26,342] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_37_model_states.pt. -[default0]:[2022-09-11 14:22:26,343] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_28-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,344] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_26_model_states.pt... -[default0]:[2022-09-11 14:22:26,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_26_model_states.pt. -[default4]:[2022-09-11 14:22:26,346] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_19-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,347] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_17_model_states.pt... -[default4]:[2022-09-11 14:22:26,352] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_17_model_states.pt. -[default0]:[2022-09-11 14:22:26,427] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_56-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,427] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_54_model_states.pt... -[default0]:[2022-09-11 14:22:26,436] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_54_model_states.pt. -[default4]:[2022-09-11 14:22:26,465] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_37-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,465] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_35_model_states.pt... -[default4]:[2022-09-11 14:22:26,472] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_35_model_states.pt. -[default4]:[2022-09-11 14:22:26,444] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_35-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,444] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_33_model_states.pt... -[default4]:[2022-09-11 14:22:26,454] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_33_model_states.pt. -[default0]:[2022-09-11 14:22:26,491] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_40-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,492] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_38_model_states.pt... -[default0]:[2022-09-11 14:22:26,495] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_38_model_states.pt. -[default0]:[2022-09-11 14:22:26,509] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_34-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,509] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_32_model_states.pt... -[default0]:[2022-09-11 14:22:26,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_32_model_states.pt. -[default0]:[2022-09-11 14:22:26,519] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_32-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,519] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_30_model_states.pt... -[default0]:[2022-09-11 14:22:26,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_30_model_states.pt. -[default0]:[2022-09-11 14:22:26,464] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_26-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,464] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_24_model_states.pt... -[default0]:[2022-09-11 14:22:26,469] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_24_model_states.pt. -[default0]:[2022-09-11 14:22:26,554] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_18-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,554] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_16_model_states.pt... -[default0]:[2022-09-11 14:22:26,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_16_model_states.pt. -[default4]:[2022-09-11 14:22:26,552] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_27-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,552] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_25_model_states.pt... -[default4]:[2022-09-11 14:22:26,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_25_model_states.pt. -[default0]:[2022-09-11 14:22:26,509] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_16-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,509] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_14_model_states.pt... -[default0]:[2022-09-11 14:22:26,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_14_model_states.pt. -[default4]:[2022-09-11 14:22:26,532] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_07-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,532] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_05_model_states.pt... -[default4]:[2022-09-11 14:22:26,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_05_model_states.pt. -[default4]:[2022-09-11 14:22:26,587] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_57-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,588] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_55_model_states.pt... -[default4]:[2022-09-11 14:22:26,590] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_55_model_states.pt. -[default4]:[2022-09-11 14:22:26,600] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_23-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,600] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_21_model_states.pt... -[default4]:[2022-09-11 14:22:26,605] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_21_model_states.pt. -[default0]:[2022-09-11 14:22:26,539] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_14-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,539] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_12_model_states.pt... -[default0]:[2022-09-11 14:22:26,554] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_12_model_states.pt. -[default0]:[2022-09-11 14:22:26,552] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_30-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,553] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_28_model_states.pt... -[default0]:[2022-09-11 14:22:26,565] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_28_model_states.pt. -[default4]:[2022-09-11 14:22:26,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_43-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,540] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_41_model_states.pt... -[default4]:[2022-09-11 14:22:26,547] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_41_model_states.pt. -[default0]:[2022-09-11 14:22:26,633] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_22-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,633] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_20_model_states.pt... -[default0]:[2022-09-11 14:22:26,635] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_20_model_states.pt. -[default4]:[2022-09-11 14:22:26,628] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_15-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,628] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_13_model_states.pt... -[default4]:[2022-09-11 14:22:26,630] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_13_model_states.pt. -[default0]:[2022-09-11 14:22:26,631] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_06-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,632] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_04_model_states.pt... -[default0]:[2022-09-11 14:22:26,634] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_04_model_states.pt. -[default0]:[2022-09-11 14:22:26,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_36-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,647] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_34_model_states.pt... -[default0]:[2022-09-11 14:22:26,652] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_34_model_states.pt. -[default0]:[2022-09-11 14:22:26,590] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_38-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,590] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_36_model_states.pt... -[default0]:[2022-09-11 14:22:26,592] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_36_model_states.pt. -[default4]:[2022-09-11 14:22:26,648] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_17-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,648] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_15_model_states.pt... -[default4]:[2022-09-11 14:22:26,651] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_15_model_states.pt. -[default4]:[2022-09-11 14:22:26,569] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_51-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,569] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_49_model_states.pt... -[default4]:[2022-09-11 14:22:26,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_49_model_states.pt. -[default4]:[2022-09-11 14:22:26,607] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_29-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,607] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_27_model_states.pt... -[default4]:[2022-09-11 14:22:26,609] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_27_model_states.pt. -[default0]:[2022-09-11 14:22:26,699] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_70-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,658] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_42-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,659] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_40_model_states.pt... -[default0]:[2022-09-11 14:22:26,661] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_40_model_states.pt. -[default4]:[2022-09-11 14:22:26,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_33-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,700] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_31_model_states.pt... -[default4]:[2022-09-11 14:22:26,702] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_31_model_states.pt. -[default0]:[2022-09-11 14:22:26,714] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_44-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,715] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_42_model_states.pt... -[default4]:[2022-09-11 14:22:26,678] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_71-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,679] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_69_model_states.pt... -[default4]:[2022-09-11 14:22:26,691] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_69_model_states.pt. -[default0]:[2022-09-11 14:22:26,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_04-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,673] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_02_model_states.pt... -[default0]:[2022-09-11 14:22:26,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_02_model_states.pt. -[default4]:[2022-09-11 14:22:26,703] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_21-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,704] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_19_model_states.pt... -[default4]:[2022-09-11 14:22:26,707] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_19_model_states.pt. -[default4]:[2022-09-11 14:22:26,754] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_45-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,754] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_43_model_states.pt... -[default4]:[2022-09-11 14:22:26,764] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_43_model_states.pt. -[default0]:[2022-09-11 14:22:26,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_10-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,676] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_08_model_states.pt... -[default0]:[2022-09-11 14:22:26,678] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_08_model_states.pt. -[default4]:[2022-09-11 14:22:26,777] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_25-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_23_model_states.pt... -[default4]:[2022-09-11 14:22:26,683] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_31-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,684] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_29_model_states.pt... -[default4]:[2022-09-11 14:22:26,687] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_29_model_states.pt. -[default0]:[2022-09-11 14:22:26,691] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_48-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_46_model_states.pt... -[default0]:[2022-09-11 14:22:26,695] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_46_model_states.pt. -[default4]:[2022-09-11 14:22:26,769] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_41-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,769] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_39_model_states.pt... -[default4]:[2022-09-11 14:22:26,771] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_39_model_states.pt. -[default0]:[2022-09-11 14:22:26,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_50-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,757] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_48_model_states.pt... -[default0]:[2022-09-11 14:22:26,760] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_48_model_states.pt. -[default0]:[2022-09-11 14:22:26,699] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_68_model_states.pt... -[default0]:[2022-09-11 14:22:26,702] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_68_model_states.pt. -[default4]:[2022-09-11 14:22:26,788] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_11-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,789] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_09_model_states.pt... -[default4]:[2022-09-11 14:22:26,790] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_09_model_states.pt. -[default4]:[2022-09-11 14:22:26,740] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_65-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,740] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_63_model_states.pt... -[default4]:[2022-09-11 14:22:26,742] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_63_model_states.pt. -[default0]:[2022-09-11 14:22:26,737] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_42_model_states.pt. -[default0]:[2022-09-11 14:22:26,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_46-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,832] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_44_model_states.pt... -[default0]:[2022-09-11 14:22:26,836] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_44_model_states.pt. -[default0]:[2022-09-11 14:22:26,779] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_20-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,779] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_18_model_states.pt... -[default0]:[2022-09-11 14:22:26,781] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_18_model_states.pt. -[default4]:[2022-09-11 14:22:26,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_49-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_47_model_states.pt... -[default4]:[2022-09-11 14:22:26,839] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_47_model_states.pt. -[default4]:[2022-09-11 14:22:26,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_53-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,856] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_51_model_states.pt... -[default4]:[2022-09-11 14:22:26,869] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_51_model_states.pt. -[default4]:[2022-09-11 14:22:26,779] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_23_model_states.pt. -[default0]:[2022-09-11 14:22:26,826] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_62-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,826] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_60_model_states.pt... -[default0]:[2022-09-11 14:22:26,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_60_model_states.pt. -[default4]:[2022-09-11 14:22:26,822] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_13-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,823] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_11_model_states.pt... -[default4]:[2022-09-11 14:22:26,825] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_11_model_states.pt. -[default4]:[2022-09-11 14:22:26,855] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_61-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,856] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_59_model_states.pt... -[default4]:[2022-09-11 14:22:26,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_59_model_states.pt. -[default4]:[2022-09-11 14:22:26,833] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_63-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,833] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_61_model_states.pt... -[default4]:[2022-09-11 14:22:26,836] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_61_model_states.pt. -[default0]:[2022-09-11 14:22:26,903] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_60-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_58_model_states.pt... -[default0]:[2022-09-11 14:22:26,905] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_58_model_states.pt. -[default4]:[2022-09-11 14:22:26,915] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_59-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,915] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_57_model_states.pt... -[default4]:[2022-09-11 14:22:26,918] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_57_model_states.pt. -[default0]:[2022-09-11 14:22:26,894] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_68-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,894] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_66_model_states.pt... -[default0]:[2022-09-11 14:22:26,897] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_66_model_states.pt. -[default0]:[2022-09-11 14:22:26,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_66-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,888] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_64_model_states.pt... -[default0]:[2022-09-11 14:22:26,891] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_64_model_states.pt. -[default4]:[2022-09-11 14:22:26,898] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_55-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,898] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_53_model_states.pt... -[default4]:[2022-09-11 14:22:26,904] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_53_model_states.pt. -[default0]:[2022-09-11 14:22:26,895] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_12-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,895] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_10_model_states.pt... -[default0]:[2022-09-11 14:22:26,897] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_10_model_states.pt. -[default4]:[2022-09-11 14:22:26,924] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_69-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,924] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_67_model_states.pt... -[default4]:[2022-09-11 14:22:26,926] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_67_model_states.pt. -[default4]:[2022-09-11 14:22:26,902] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_67-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_65_model_states.pt... -[default4]:[2022-09-11 14:22:26,904] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_65_model_states.pt. -[default4]:[2022-09-11 14:22:26,939] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_47-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:26,940] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_45_model_states.pt... -[default4]:[2022-09-11 14:22:26,942] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_45_model_states.pt. -[default0]:[2022-09-11 14:22:26,926] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_52-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,927] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_50_model_states.pt... -[default0]:[2022-09-11 14:22:26,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_50_model_states.pt. -[default0]:[2022-09-11 14:22:26,937] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_54-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,938] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_52_model_states.pt... -[default0]:[2022-09-11 14:22:26,940] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_52_model_states.pt. -[default0]:[2022-09-11 14:22:26,995] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_58-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,995] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_56_model_states.pt... -[default0]:[2022-09-11 14:22:26,997] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_56_model_states.pt. -[default0]:[2022-09-11 14:22:26,998] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_64-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:26,998] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_62_model_states.pt... -[default0]:[2022-09-11 14:22:27,000] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_62_model_states.pt. -[default4]:[2022-09-11 14:22:27,059] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_03-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:27,059] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_01_model_states.pt... -[default4]:[2022-09-11 14:22:27,061] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_01_model_states.pt. -[default4]:[2022-09-11 14:22:27,181] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_09-model_00-model_states.pt. -[default4]:[2022-09-11 14:22:27,181] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_07_model_states.pt... -[default0]:[2022-09-11 14:22:27,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_08-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:27,156] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_06_model_states.pt... -[default0]:[2022-09-11 14:22:27,160] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_06_model_states.pt. -[default4]:[2022-09-11 14:22:27,184] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_07_model_states.pt. -[default0]:[2022-09-11 14:22:28,078] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/layer_01-model_00-model_states.pt. -[default0]:[2022-09-11 14:22:28,079] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_00_model_states.pt -[default0]:[2022-09-11 14:22:28,079] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_00_model_states.pt... -[default0]:[2022-09-11 14:22:28,080] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/mp_rank_00_model_states.pt. -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default0]:[2022-09-11 14:22:28,187] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default6]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default3]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default1]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default5]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default0]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default2]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default7]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default4]:[2022-09-11 14:22:28,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default1]:[2022-09-11 14:22:36,594] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-11 14:22:36,595] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default5]:[2022-09-11 14:22:36,931] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-11 14:22:36,931] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default0]:[2022-09-11 14:22:37,040] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-11 14:22:37,040] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default2]:[2022-09-11 14:22:37,083] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-11 14:22:37,084] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default3]:[2022-09-11 14:22:37,158] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-11 14:22:37,158] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default3]:[2022-09-11 14:22:37,244] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-11 14:22:37,244] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default1]:[2022-09-11 14:22:37,441] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-11 14:22:37,441] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default3]:[2022-09-11 14:22:37,451] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-11 14:22:37,451] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default2]:[2022-09-11 14:22:37,464] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-11 14:22:37,464] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default0]:[2022-09-11 14:22:37,583] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-11 14:22:37,583] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default0]:[2022-09-11 14:22:37,533] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-11 14:22:37,533] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default3]:[2022-09-11 14:22:37,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-11 14:22:37,615] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default4]:[2022-09-11 14:22:37,628] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-11 14:22:37,628] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default0]:[2022-09-11 14:22:37,688] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-11 14:22:37,688] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default3]:[2022-09-11 14:22:37,767] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-11 14:22:37,767] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default3]:[2022-09-11 14:22:37,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-11 14:22:37,757] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default3]:[2022-09-11 14:22:37,830] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-11 14:22:37,831] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default3]:[2022-09-11 14:22:37,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-11 14:22:37,933] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default0]:[2022-09-11 14:22:38,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-11 14:22:38,020] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default1]:[2022-09-11 14:22:38,076] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-11 14:22:38,076] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default2]:[2022-09-11 14:22:38,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-11 14:22:38,020] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default2]:[2022-09-11 14:22:38,123] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-11 14:22:38,123] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default1]:[2022-09-11 14:22:38,076] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-11 14:22:38,076] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default3]:[2022-09-11 14:22:38,254] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-11 14:22:38,254] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default6]:[2022-09-11 14:22:38,205] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-11 14:22:38,205] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default1]:[2022-09-11 14:22:38,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-11 14:22:38,284] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default5]:[2022-09-11 14:22:38,241] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-11 14:22:38,241] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default0]:[2022-09-11 14:22:38,237] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-11 14:22:38,237] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default5]:[2022-09-11 14:22:38,250] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-11 14:22:38,250] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default3]:[2022-09-11 14:22:38,344] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-11 14:22:38,344] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default6]:[2022-09-11 14:22:38,344] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-11 14:22:38,345] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default6]:[2022-09-11 14:22:38,371] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-11 14:22:38,371] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default3]:[2022-09-11 14:22:38,377] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-11 14:22:38,377] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default6]:[2022-09-11 14:22:38,435] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-11 14:22:38,435] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default6]:[2022-09-11 14:22:38,478] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-11 14:22:38,478] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default2]:[2022-09-11 14:22:38,558] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-11 14:22:38,559] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default7]:[2022-09-11 14:22:38,570] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-11 14:22:38,570] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default0]:[2022-09-11 14:22:38,556] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-11 14:22:38,556] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default5]:[2022-09-11 14:22:38,565] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-11 14:22:38,565] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default2]:[2022-09-11 14:22:38,601] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-11 14:22:38,601] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default6]:[2022-09-11 14:22:38,644] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-11 14:22:38,644] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default6]:[2022-09-11 14:22:38,616] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-11 14:22:38,616] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default2]:[2022-09-11 14:22:38,663] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-11 14:22:38,663] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default4]:[2022-09-11 14:22:38,586] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-11 14:22:38,587] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default4]:[2022-09-11 14:22:38,662] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-11 14:22:38,662] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default7]:[2022-09-11 14:22:38,630] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-11 14:22:38,630] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default6]:[2022-09-11 14:22:38,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-11 14:22:38,728] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default6]:[2022-09-11 14:22:38,693] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-11 14:22:38,693] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default7]:[2022-09-11 14:22:38,683] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-11 14:22:38,684] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default4]:[2022-09-11 14:22:38,771] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-11 14:22:38,771] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default2]:[2022-09-11 14:22:38,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-11 14:22:38,728] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default4]:[2022-09-11 14:22:38,763] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-11 14:22:38,763] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default2]:[2022-09-11 14:22:38,742] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-11 14:22:38,742] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default0]:[2022-09-11 14:22:38,809] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-11 14:22:38,809] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default1]:[2022-09-11 14:22:38,812] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-11 14:22:38,812] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default5]:[2022-09-11 14:22:38,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-11 14:22:38,783] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default2]:[2022-09-11 14:22:38,754] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-11 14:22:38,754] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default1]:[2022-09-11 14:22:38,806] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-11 14:22:38,806] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default3]:[2022-09-11 14:22:38,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-11 14:22:38,797] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default6]:[2022-09-11 14:22:38,822] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-11 14:22:38,822] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default7]:[2022-09-11 14:22:38,830] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-11 14:22:38,831] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default4]:[2022-09-11 14:22:38,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-11 14:22:38,888] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default3]:[2022-09-11 14:22:38,914] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-11 14:22:38,915] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default7]:[2022-09-11 14:22:38,839] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-11 14:22:38,839] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default0]:[2022-09-11 14:22:38,924] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-11 14:22:38,924] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default6]:[2022-09-11 14:22:38,863] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-11 14:22:38,863] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default2]:[2022-09-11 14:22:38,959] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-11 14:22:38,959] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default5]:[2022-09-11 14:22:38,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-11 14:22:38,981] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default4]:[2022-09-11 14:22:38,970] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-11 14:22:38,970] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default7]:[2022-09-11 14:22:38,960] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-11 14:22:38,960] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default1]:[2022-09-11 14:22:38,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-11 14:22:38,993] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default5]:[2022-09-11 14:22:39,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-11 14:22:39,016] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default7]:[2022-09-11 14:22:38,988] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-11 14:22:38,988] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default5]:[2022-09-11 14:22:39,000] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-11 14:22:39,000] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default4]:[2022-09-11 14:22:39,000] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-11 14:22:39,001] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default5]:[2022-09-11 14:22:39,071] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-11 14:22:39,071] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default3]:[2022-09-11 14:22:39,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-11 14:22:39,023] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default5]:[2022-09-11 14:22:39,053] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-11 14:22:39,053] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default4]:[2022-09-11 14:22:39,082] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-11 14:22:39,082] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default0]:[2022-09-11 14:22:39,074] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-11 14:22:39,074] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default1]:[2022-09-11 14:22:39,128] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-11 14:22:39,128] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default1]:[2022-09-11 14:22:39,092] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-11 14:22:39,092] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default3]:[2022-09-11 14:22:39,219] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-11 14:22:39,219] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default7]:[2022-09-11 14:22:39,143] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-11 14:22:39,143] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default7]:[2022-09-11 14:22:39,166] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-11 14:22:39,166] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default5]:[2022-09-11 14:22:39,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-11 14:22:39,203] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default7]:[2022-09-11 14:22:39,204] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-11 14:22:39,204] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default4]:[2022-09-11 14:22:39,201] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-11 14:22:39,201] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default1]:[2022-09-11 14:22:39,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-11 14:22:39,276] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default5]:[2022-09-11 14:22:39,242] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-11 14:22:39,243] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default7]:[2022-09-11 14:22:39,272] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-11 14:22:39,272] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default1]:[2022-09-11 14:22:39,261] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-11 14:22:39,261] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default3]:[2022-09-11 14:22:39,294] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-11 14:22:39,295] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default6]:[2022-09-11 14:22:39,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-11 14:22:39,276] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default5]:[2022-09-11 14:22:39,357] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-11 14:22:39,357] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default7]:[2022-09-11 14:22:39,371] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-11 14:22:39,371] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default7]:[2022-09-11 14:22:39,312] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-11 14:22:39,312] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default1]:[2022-09-11 14:22:39,286] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-11 14:22:39,287] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default2]:[2022-09-11 14:22:39,333] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-11 14:22:39,333] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default7]:[2022-09-11 14:22:39,330] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-11 14:22:39,330] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default4]:[2022-09-11 14:22:39,369] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-11 14:22:39,369] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default5]:[2022-09-11 14:22:39,337] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-11 14:22:39,337] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default4]:[2022-09-11 14:22:39,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-11 14:22:39,385] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default5]:[2022-09-11 14:22:39,415] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-11 14:22:39,416] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default2]:[2022-09-11 14:22:39,428] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-11 14:22:39,428] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default2]:[2022-09-11 14:22:39,437] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-11 14:22:39,437] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default1]:[2022-09-11 14:22:39,460] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-11 14:22:39,460] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default4]:[2022-09-11 14:22:39,413] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-11 14:22:39,414] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default6]:[2022-09-11 14:22:39,436] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-11 14:22:39,436] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default3]:[2022-09-11 14:22:39,462] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-11 14:22:39,462] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default5]:[2022-09-11 14:22:39,452] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-11 14:22:39,452] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default0]:[2022-09-11 14:22:39,560] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-11 14:22:39,560] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default3]:[2022-09-11 14:22:39,547] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-11 14:22:39,547] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default5]:[2022-09-11 14:22:39,545] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-11 14:22:39,545] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default5]:[2022-09-11 14:22:39,590] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-11 14:22:39,590] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default0]:[2022-09-11 14:22:39,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-11 14:22:39,596] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default1]:[2022-09-11 14:22:39,600] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-11 14:22:39,600] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default1]:[2022-09-11 14:22:39,589] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-11 14:22:39,589] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default0]:[2022-09-11 14:22:39,656] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-11 14:22:39,656] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default0]:[2022-09-11 14:22:39,612] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-11 14:22:39,612] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default3]:[2022-09-11 14:22:39,590] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-11 14:22:39,590] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default4]:[2022-09-11 14:22:39,581] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-11 14:22:39,581] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default0]:[2022-09-11 14:22:39,613] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-11 14:22:39,613] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default5]:[2022-09-11 14:22:39,635] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-11 14:22:39,635] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default7]:[2022-09-11 14:22:39,623] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-11 14:22:39,623] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default2]:[2022-09-11 14:22:39,711] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-11 14:22:39,711] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default4]:[2022-09-11 14:22:39,652] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-11 14:22:39,652] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default7]:[2022-09-11 14:22:39,711] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-11 14:22:39,711] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default4]:[2022-09-11 14:22:39,671] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-11 14:22:39,671] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default3]:[2022-09-11 14:22:39,679] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-11 14:22:39,679] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default1]:[2022-09-11 14:22:39,735] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-11 14:22:39,735] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default6]:[2022-09-11 14:22:39,767] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-11 14:22:39,767] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default0]:[2022-09-11 14:22:39,754] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-11 14:22:39,755] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default3]:[2022-09-11 14:22:39,727] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-11 14:22:39,727] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default1]:[2022-09-11 14:22:39,802] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-11 14:22:39,802] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default2]:[2022-09-11 14:22:39,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-11 14:22:39,820] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default1]:[2022-09-11 14:22:39,748] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-11 14:22:39,748] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default0]:[2022-09-11 14:22:39,761] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-11 14:22:39,761] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default0]:[2022-09-11 14:22:39,765] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-11 14:22:39,765] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default0]:[2022-09-11 14:22:39,864] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-11 14:22:39,864] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default7]:[2022-09-11 14:22:39,799] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-11 14:22:39,799] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default6]:[2022-09-11 14:22:39,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-11 14:22:39,865] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default1]:[2022-09-11 14:22:39,814] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-11 14:22:39,814] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default1]:[2022-09-11 14:22:39,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-11 14:22:39,870] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default2]:[2022-09-11 14:22:39,874] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-11 14:22:39,874] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default2]:[2022-09-11 14:22:39,905] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-11 14:22:39,905] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default6]:[2022-09-11 14:22:39,945] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-11 14:22:39,945] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default0]:[2022-09-11 14:22:39,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-11 14:22:39,932] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default3]:[2022-09-11 14:22:39,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-11 14:22:39,973] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default5]:[2022-09-11 14:22:39,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-11 14:22:39,947] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default7]:[2022-09-11 14:22:39,959] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-11 14:22:39,959] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default5]:[2022-09-11 14:22:39,971] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-11 14:22:39,971] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default0]:[2022-09-11 14:22:40,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-11 14:22:40,037] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default5]:[2022-09-11 14:22:39,982] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-11 14:22:39,983] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default5]:[2022-09-11 14:22:40,088] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-11 14:22:40,088] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default4]:[2022-09-11 14:22:40,081] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-11 14:22:40,081] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default7]:[2022-09-11 14:22:40,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-11 14:22:40,025] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default3]:[2022-09-11 14:22:40,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-11 14:22:40,079] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default2]:[2022-09-11 14:22:40,067] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-11 14:22:40,067] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default7]:[2022-09-11 14:22:40,124] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-11 14:22:40,124] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default6]:[2022-09-11 14:22:40,107] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-11 14:22:40,108] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default6]:[2022-09-11 14:22:40,187] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-11 14:22:40,187] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default3]:[2022-09-11 14:22:40,180] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-11 14:22:40,180] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default4]:[2022-09-11 14:22:40,167] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-11 14:22:40,167] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default6]:[2022-09-11 14:22:40,252] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-11 14:22:40,252] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default1]:[2022-09-11 14:22:40,215] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-11 14:22:40,216] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default0]:[2022-09-11 14:22:40,239] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-11 14:22:40,239] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default2]:[2022-09-11 14:22:40,277] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-11 14:22:40,277] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default4]:[2022-09-11 14:22:40,249] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-11 14:22:40,249] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default4]:[2022-09-11 14:22:40,218] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-11 14:22:40,218] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default7]:[2022-09-11 14:22:40,252] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-11 14:22:40,252] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default0]:[2022-09-11 14:22:40,300] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-11 14:22:40,300] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default6]:[2022-09-11 14:22:40,264] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-11 14:22:40,265] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default6]:[2022-09-11 14:22:40,279] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-11 14:22:40,279] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default1]:[2022-09-11 14:22:40,341] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-11 14:22:40,341] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default3]:[2022-09-11 14:22:40,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-11 14:22:40,284] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default6]:[2022-09-11 14:22:40,299] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-11 14:22:40,299] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default0]:[2022-09-11 14:22:40,323] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-11 14:22:40,323] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default2]:[2022-09-11 14:22:40,372] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-11 14:22:40,372] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default0]:[2022-09-11 14:22:40,418] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-11 14:22:40,418] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default4]:[2022-09-11 14:22:40,404] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-11 14:22:40,404] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default2]:[2022-09-11 14:22:40,406] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-11 14:22:40,406] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default7]:[2022-09-11 14:22:40,419] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-11 14:22:40,419] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default2]:[2022-09-11 14:22:40,504] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-11 14:22:40,504] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default0]:[2022-09-11 14:22:40,482] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-11 14:22:40,482] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default5]:[2022-09-11 14:22:40,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-11 14:22:40,455] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default7]:[2022-09-11 14:22:40,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-11 14:22:40,559] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default2]:[2022-09-11 14:22:40,563] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-11 14:22:40,563] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default4]:[2022-09-11 14:22:40,613] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-11 14:22:40,613] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default1]:[2022-09-11 14:22:40,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-11 14:22:40,579] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default7]:[2022-09-11 14:22:40,599] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-11 14:22:40,599] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default1]:[2022-09-11 14:22:40,688] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-11 14:22:40,688] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default7]:[2022-09-11 14:22:40,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-11 14:22:40,857] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default5]:[2022-09-11 14:22:40,882] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-11 14:22:40,882] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default2]:[2022-09-11 14:22:40,878] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-11 14:22:40,879] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default6]:[2022-09-11 14:22:40,911] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-11 14:22:40,911] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default3]:[2022-09-11 14:22:40,961] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-11 14:22:40,961] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default5]:[2022-09-11 14:22:41,064] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-11 14:22:41,064] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default7]:[2022-09-11 14:22:41,055] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-11 14:22:41,056] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default4]:[2022-09-11 14:22:41,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-11 14:22:41,043] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default4]:[2022-09-11 14:22:40,995] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-11 14:22:40,996] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default6]:[2022-09-11 14:22:41,038] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-11 14:22:41,038] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default2]:[2022-09-11 14:22:41,060] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-11 14:22:41,060] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default1]:[2022-09-11 14:22:41,145] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-11 14:22:41,145] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default7]:[2022-09-11 14:22:41,134] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-11 14:22:41,134] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default4]:[2022-09-11 14:22:41,130] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-11 14:22:41,130] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default1]:[2022-09-11 14:22:41,182] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-11 14:22:41,182] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default4]:[2022-09-11 14:22:41,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-11 14:22:41,262] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default6]:[2022-09-11 14:22:41,329] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-11 14:22:41,329] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default2]:[2022-09-11 14:22:41,360] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-11 14:22:41,360] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default6]:[2022-09-11 14:22:41,331] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-11 14:22:41,331] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default6]:[2022-09-11 14:22:41,364] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-11 14:22:41,364] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default3]:[2022-09-11 14:22:41,415] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-11 14:22:41,415] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default4]:[2022-09-11 14:22:41,428] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-11 14:22:41,428] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default0]:[2022-09-11 14:22:41,440] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-11 14:22:41,441] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default5]:[2022-09-11 14:22:41,476] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-11 14:22:41,476] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default4]:[2022-09-11 14:22:41,466] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-11 14:22:41,466] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default2]:[2022-09-11 14:22:41,509] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-11 14:22:41,509] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default6]:[2022-09-11 14:22:41,562] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-11 14:22:41,562] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default7]:[2022-09-11 14:22:41,507] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-11 14:22:41,507] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default1]:[2022-09-11 14:22:41,596] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-11 14:22:41,596] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default3]:[2022-09-11 14:22:41,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-11 14:22:41,728] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default5]:[2022-09-11 14:22:42,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-11 14:22:42,028] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default0]:[2022-09-11 14:22:41,935] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-11 14:22:41,935] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default0]:[2022-09-11 14:22:42,292] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-11 14:22:42,292] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default3]:[2022-09-11 14:22:42,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-11 14:22:42,540] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default0]:[2022-09-11 14:22:42,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-11 14:22:42,528] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default6]:[2022-09-11 14:22:42,547] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-11 14:22:42,547] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default2]:[2022-09-11 14:22:42,623] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-11 14:22:42,623] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default7]:[2022-09-11 14:22:42,751] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-11 14:22:42,751] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default1]:[2022-09-11 14:22:42,726] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-11 14:22:42,727] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default5]:[2022-09-11 14:22:42,868] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-11 14:22:42,868] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default1]:[2022-09-11 14:22:43,388] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-11 14:22:43,388] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default4]:[2022-09-11 14:22:43,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-11 14:22:43,675] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default2]:[2022-09-11 14:22:43,650] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-11 14:22:43,650] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default3]:[2022-09-11 14:22:43,824] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-11 14:22:43,824] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default0]:[2022-09-11 14:22:44,393] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-11 14:22:44,394] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default1]:[2022-09-11 14:22:44,451] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-11 14:22:44,451] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default5]:[2022-09-11 14:22:45,060] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-11 14:22:45,060] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default7]:[2022-09-11 14:22:45,359] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-11 14:22:45,359] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default5]:[2022-09-11 14:22:45,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-11 14:22:45,417] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default3]:[2022-09-11 14:22:45,442] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-11 14:22:45,442] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default2]:[2022-09-11 14:22:45,454] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-11 14:22:45,454] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default4]:[2022-09-11 14:22:45,480] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-11 14:22:45,480] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default4]:[2022-09-11 14:22:45,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-11 14:22:45,595] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default0]:[2022-09-11 14:22:45,679] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-11 14:22:45,680] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default5]:[2022-09-11 14:22:45,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-11 14:22:45,775] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default1]:[2022-09-11 14:22:45,727] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-11 14:22:45,727] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default7]:[2022-09-11 14:22:45,912] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-11 14:22:45,912] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default4]:[2022-09-11 14:22:46,193] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-11 14:22:46,193] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default2]:[2022-09-11 14:22:46,183] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-11 14:22:46,183] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default3]:[2022-09-11 14:22:46,235] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-11 14:22:46,235] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default6]:[2022-09-11 14:22:46,354] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-11 14:22:46,354] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default4]:[2022-09-11 14:22:46,443] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-11 14:22:46,443] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default5]:[2022-09-11 14:22:46,467] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-11 14:22:46,467] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default6]:[2022-09-11 14:22:47,361] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-11 14:22:47,362] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default7]:[2022-09-11 14:22:47,683] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-11 14:22:47,683] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default2]:[2022-09-11 14:22:47,776] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-11 14:22:47,776] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default3]:[2022-09-11 14:22:48,064] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-11 14:22:48,064] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default2]:[2022-09-11 14:22:48,181] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-11 14:22:48,181] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default3]:[2022-09-11 14:22:48,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-11 14:22:48,576] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default6]:[2022-09-11 14:22:48,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-11 14:22:48,636] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default1]:[2022-09-11 14:22:48,611] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-11 14:22:48,611] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default6]:[2022-09-11 14:22:48,718] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-11 14:22:48,718] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default0]:[2022-09-11 14:22:48,631] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-11 14:22:48,632] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default4]:[2022-09-11 14:22:48,998] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-11 14:22:48,999] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default3]:[2022-09-11 14:22:49,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-11 14:22:49,028] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default5]:[2022-09-11 14:22:49,306] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-11 14:22:49,306] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default7]:[2022-09-11 14:22:49,761] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-11 14:22:49,762] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default0]:[2022-09-11 14:22:49,746] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-11 14:22:49,746] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default2]:[2022-09-11 14:22:49,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-11 14:22:49,837] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default4]:[2022-09-11 14:22:50,457] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-11 14:22:50,458] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default5]:[2022-09-11 14:22:50,705] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-11 14:22:50,705] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default6]:[2022-09-11 14:22:50,849] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-11 14:22:50,850] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default7]:[2022-09-11 14:22:50,916] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-11 14:22:50,916] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default6]:[2022-09-11 14:22:51,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-11 14:22:51,276] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default1]:[2022-09-11 14:22:51,511] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-11 14:22:51,511] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default7]:[2022-09-11 14:22:51,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-11 14:22:51,540] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default0]:[2022-09-11 14:22:51,692] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-11 14:22:51,692] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default1]:[2022-09-11 14:22:51,717] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-11 14:22:51,717] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default2]:[2022-09-11 14:22:55,356] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-11 14:22:55,356] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default3]:[2022-09-11 14:22:55,431] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-11 14:22:55,431] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default4]:[2022-09-11 14:22:55,470] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-11 14:22:55,470] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default7]:[2022-09-11 14:22:56,264] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-11 14:22:56,264] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default6]:[2022-09-11 14:22:56,232] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-11 14:22:56,232] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default5]:[2022-09-11 14:22:56,515] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-11 14:22:56,515] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default0]:[2022-09-11 14:23:00,400] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-11 14:23:00,402] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,532] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-11 14:23:00,532] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2241/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default1]:[2022-09-11 14:23:00,532] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,532] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]: successfully saved checkpoint at iteration 2241 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:time (ms) | save-checkpoint: 37777.73 -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default2]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default3]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default1]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default6]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default0]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default5]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default4]:[2022-09-11 14:23:00,533] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2241 is ready now! -[default7]: iteration 2242/ 3100 | consumed samples: 4591616 | consumed tokens: 9403629568 | elapsed time per iteration (s): 179.25 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.218214E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.425 | TFLOPs: 116.63 | -[default7]: iteration 2243/ 3100 | consumed samples: 4593664 | consumed tokens: 9407823872 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.311590E-01 | grad norm: 0.361 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 2244/ 3100 | consumed samples: 4595712 | consumed tokens: 9412018176 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.164357E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 2245/ 3100 | consumed samples: 4597760 | consumed tokens: 9416212480 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.266891E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 2246/ 3100 | consumed samples: 4599808 | consumed tokens: 9420406784 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.256296E-01 | grad norm: 0.327 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 2247/ 3100 | consumed samples: 4601856 | consumed tokens: 9424601088 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.264699E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 2248/ 3100 | consumed samples: 4603904 | consumed tokens: 9428795392 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.102385E-01 | grad norm: 0.333 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.72 | -[default7]: iteration 2249/ 3100 | consumed samples: 4605952 | consumed tokens: 9432989696 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.173700E-01 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.50 | -[default7]: iteration 2250/ 3100 | consumed samples: 4608000 | consumed tokens: 9437184000 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.279100E-01 | grad norm: 0.650 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]:validation_pretraining loss at iteration 2250 | lm loss value: 2.481650E+00 | lm loss PPL: 1.196099E+01 | -[default7]:----------------------------------------------------------------------------------------------------------- -[default7]: iteration 2251/ 3100 | consumed samples: 4610048 | consumed tokens: 9441378304 | elapsed time per iteration (s): 183.15 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.209216E-01 | grad norm: 0.337 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 11.182 | TFLOPs: 114.15 | -[default7]: iteration 2252/ 3100 | consumed samples: 4612096 | consumed tokens: 9445572608 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.104700E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.76 | -[default7]: iteration 2253/ 3100 | consumed samples: 4614144 | consumed tokens: 9449766912 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.283950E-01 | grad norm: 0.352 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.50 | -[default7]: iteration 2254/ 3100 | consumed samples: 4616192 | consumed tokens: 9453961216 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.220662E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.56 | -[default7]: iteration 2255/ 3100 | consumed samples: 4618240 | consumed tokens: 9458155520 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.129171E-01 | grad norm: 0.327 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 2256/ 3100 | consumed samples: 4620288 | consumed tokens: 9462349824 | elapsed time per iteration (s): 140.35 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.163242E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.592 | TFLOPs: 148.96 | -[default7]: iteration 2257/ 3100 | consumed samples: 4622336 | consumed tokens: 9466544128 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.245076E-01 | grad norm: 0.335 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 2258/ 3100 | consumed samples: 4624384 | consumed tokens: 9470738432 | elapsed time per iteration (s): 141.83 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.269560E-01 | grad norm: 0.335 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 2259/ 3100 | consumed samples: 4626432 | consumed tokens: 9474932736 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.046137E-01 | grad norm: 0.324 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.468 | TFLOPs: 147.69 | -[default7]: iteration 2260/ 3100 | consumed samples: 4628480 | consumed tokens: 9479127040 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.198241E-01 | grad norm: 0.475 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 2261/ 3100 | consumed samples: 4630528 | consumed tokens: 9483321344 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.008591E-01 | grad norm: 0.353 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 2262/ 3100 | consumed samples: 4632576 | consumed tokens: 9487515648 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.186331E-01 | grad norm: 0.370 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.57 | -[default7]: iteration 2263/ 3100 | consumed samples: 4634624 | consumed tokens: 9491709952 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.196957E-01 | grad norm: 0.371 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.62 | -[default7]: iteration 2264/ 3100 | consumed samples: 4636672 | consumed tokens: 9495904256 | elapsed time per iteration (s): 140.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.142227E-01 | grad norm: 0.340 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.542 | TFLOPs: 148.45 | -[default7]: iteration 2265/ 3100 | consumed samples: 4638720 | consumed tokens: 9500098560 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.233789E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 2266/ 3100 | consumed samples: 4640768 | consumed tokens: 9504292864 | elapsed time per iteration (s): 140.84 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.159641E-01 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.541 | TFLOPs: 148.44 | -[default7]: iteration 2267/ 3100 | consumed samples: 4642816 | consumed tokens: 9508487168 | elapsed time per iteration (s): 141.31 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.078866E-01 | grad norm: 0.442 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.493 | TFLOPs: 147.95 | -[default7]: iteration 2268/ 3100 | consumed samples: 4644864 | consumed tokens: 9512681472 | elapsed time per iteration (s): 140.98 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.078810E-01 | grad norm: 0.391 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.527 | TFLOPs: 148.30 | -[default7]: iteration 2269/ 3100 | consumed samples: 4646912 | consumed tokens: 9516875776 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.113289E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 2270/ 3100 | consumed samples: 4648960 | consumed tokens: 9521070080 | elapsed time per iteration (s): 141.97 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.307301E-01 | grad norm: 0.388 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.426 | TFLOPs: 147.27 | -[default7]: iteration 2271/ 3100 | consumed samples: 4651008 | consumed tokens: 9525264384 | elapsed time per iteration (s): 141.24 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.187244E-01 | grad norm: 0.332 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.500 | TFLOPs: 148.02 | -[default7]: iteration 2272/ 3100 | consumed samples: 4653056 | consumed tokens: 9529458688 | elapsed time per iteration (s): 140.90 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.253881E-01 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.535 | TFLOPs: 148.38 | -[default7]: iteration 2273/ 3100 | consumed samples: 4655104 | consumed tokens: 9533652992 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.199022E-01 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 2274/ 3100 | consumed samples: 4657152 | consumed tokens: 9537847296 | elapsed time per iteration (s): 140.33 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.088293E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.594 | TFLOPs: 148.98 | -[default7]: iteration 2275/ 3100 | consumed samples: 4659200 | consumed tokens: 9542041600 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.216861E-01 | grad norm: 0.334 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 2276/ 3100 | consumed samples: 4661248 | consumed tokens: 9546235904 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.246010E-01 | grad norm: 0.349 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.52 | -[default7]: iteration 2277/ 3100 | consumed samples: 4663296 | consumed tokens: 9550430208 | elapsed time per iteration (s): 141.44 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.128927E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 2278/ 3100 | consumed samples: 4665344 | consumed tokens: 9554624512 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.156290E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 2279/ 3100 | consumed samples: 4667392 | consumed tokens: 9558818816 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.148613E-01 | grad norm: 0.365 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 2280/ 3100 | consumed samples: 4669440 | consumed tokens: 9563013120 | elapsed time per iteration (s): 140.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.116303E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.568 | TFLOPs: 148.72 | -[default7]: iteration 2281/ 3100 | consumed samples: 4671488 | consumed tokens: 9567207424 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.068393E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 2282/ 3100 | consumed samples: 4673536 | consumed tokens: 9571401728 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 6.982375E-01 | grad norm: 0.345 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 2283/ 3100 | consumed samples: 4675584 | consumed tokens: 9575596032 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.213265E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 2284/ 3100 | consumed samples: 4677632 | consumed tokens: 9579790336 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.169167E-01 | grad norm: 0.336 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 2285/ 3100 | consumed samples: 4679680 | consumed tokens: 9583984640 | elapsed time per iteration (s): 141.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.127616E-01 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.456 | TFLOPs: 147.58 | -[default7]: iteration 2286/ 3100 | consumed samples: 4681728 | consumed tokens: 9588178944 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.010197E-01 | grad norm: 0.333 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 2287/ 3100 | consumed samples: 4683776 | consumed tokens: 9592373248 | elapsed time per iteration (s): 141.48 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.055951E-01 | grad norm: 0.352 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 2288/ 3100 | consumed samples: 4685824 | consumed tokens: 9596567552 | elapsed time per iteration (s): 140.42 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.112931E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.585 | TFLOPs: 148.89 | -[default7]: iteration 2289/ 3100 | consumed samples: 4687872 | consumed tokens: 9600761856 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.292395E-01 | grad norm: 0.352 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 2290/ 3100 | consumed samples: 4689920 | consumed tokens: 9604956160 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.219231E-01 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.438 | TFLOPs: 147.39 | -[default7]: iteration 2291/ 3100 | consumed samples: 4691968 | consumed tokens: 9609150464 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.229713E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 2292/ 3100 | consumed samples: 4694016 | consumed tokens: 9613344768 | elapsed time per iteration (s): 141.82 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.177294E-01 | grad norm: 0.339 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.440 | TFLOPs: 147.41 | -[default7]: iteration 2293/ 3100 | consumed samples: 4696064 | consumed tokens: 9617539072 | elapsed time per iteration (s): 142.05 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.219059E-01 | grad norm: 0.335 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.417 | TFLOPs: 147.18 | -[default7]: iteration 2294/ 3100 | consumed samples: 4698112 | consumed tokens: 9621733376 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.240582E-01 | grad norm: 0.351 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 2295/ 3100 | consumed samples: 4700160 | consumed tokens: 9625927680 | elapsed time per iteration (s): 140.01 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.181476E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.628 | TFLOPs: 149.33 | -[default7]: iteration 2296/ 3100 | consumed samples: 4702208 | consumed tokens: 9630121984 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.064927E-01 | grad norm: 0.337 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.57 | -[default7]: iteration 2297/ 3100 | consumed samples: 4704256 | consumed tokens: 9634316288 | elapsed time per iteration (s): 141.50 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.279448E-01 | grad norm: 0.337 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.474 | TFLOPs: 147.75 | -[default7]: iteration 2298/ 3100 | consumed samples: 4706304 | consumed tokens: 9638510592 | elapsed time per iteration (s): 141.53 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.161689E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.471 | TFLOPs: 147.73 | -[default7]: iteration 2299/ 3100 | consumed samples: 4708352 | consumed tokens: 9642704896 | elapsed time per iteration (s): 140.40 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.197827E-01 | grad norm: 0.366 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.587 | TFLOPs: 148.91 | -[default7]: iteration 2300/ 3100 | consumed samples: 4710400 | consumed tokens: 9646899200 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.200949E-01 | grad norm: 0.339 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 2301/ 3100 | consumed samples: 4712448 | consumed tokens: 9651093504 | elapsed time per iteration (s): 141.79 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.127034E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.444 | TFLOPs: 147.45 | -[default7]: iteration 2302/ 3100 | consumed samples: 4714496 | consumed tokens: 9655287808 | elapsed time per iteration (s): 140.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.142619E-01 | grad norm: 0.335 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.569 | TFLOPs: 148.73 | -[default7]: iteration 2303/ 3100 | consumed samples: 4716544 | consumed tokens: 9659482112 | elapsed time per iteration (s): 140.67 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.136294E-01 | grad norm: 0.354 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.559 | TFLOPs: 148.63 | -[default7]: iteration 2304/ 3100 | consumed samples: 4718592 | consumed tokens: 9663676416 | elapsed time per iteration (s): 141.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 6.996458E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.462 | TFLOPs: 147.64 | -[default7]: iteration 2305/ 3100 | consumed samples: 4720640 | consumed tokens: 9667870720 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.262349E-01 | grad norm: 0.325 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 2306/ 3100 | consumed samples: 4722688 | consumed tokens: 9672065024 | elapsed time per iteration (s): 141.72 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.034969E-01 | grad norm: 0.328 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.451 | TFLOPs: 147.53 | -[default7]: iteration 2307/ 3100 | consumed samples: 4724736 | consumed tokens: 9676259328 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.144470E-01 | grad norm: 0.337 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.459 | TFLOPs: 147.60 | -[default7]: iteration 2308/ 3100 | consumed samples: 4726784 | consumed tokens: 9680453632 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.097460E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.48 | -[default7]: iteration 2309/ 3100 | consumed samples: 4728832 | consumed tokens: 9684647936 | elapsed time per iteration (s): 141.65 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.131855E-01 | grad norm: 0.327 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.458 | TFLOPs: 147.59 | -[default7]: iteration 2310/ 3100 | consumed samples: 4730880 | consumed tokens: 9688842240 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.124377E-01 | grad norm: 0.334 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.50 | -[default7]: iteration 2311/ 3100 | consumed samples: 4732928 | consumed tokens: 9693036544 | elapsed time per iteration (s): 141.76 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.022817E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.447 | TFLOPs: 147.48 | -[default7]: iteration 2312/ 3100 | consumed samples: 4734976 | consumed tokens: 9697230848 | elapsed time per iteration (s): 141.77 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 6.888589E-01 | grad norm: 0.341 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.446 | TFLOPs: 147.48 | -[default7]: iteration 2313/ 3100 | consumed samples: 4737024 | consumed tokens: 9701425152 | elapsed time per iteration (s): 141.22 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.159981E-01 | grad norm: 0.337 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.503 | TFLOPs: 148.05 | -[default7]: iteration 2314/ 3100 | consumed samples: 4739072 | consumed tokens: 9705619456 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.108971E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 2315/ 3100 | consumed samples: 4741120 | consumed tokens: 9709813760 | elapsed time per iteration (s): 141.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.179123E-01 | grad norm: 0.340 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.475 | TFLOPs: 147.77 | -[default7]: iteration 2316/ 3100 | consumed samples: 4743168 | consumed tokens: 9714008064 | elapsed time per iteration (s): 141.54 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.193682E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.470 | TFLOPs: 147.71 | -[default7]: iteration 2317/ 3100 | consumed samples: 4745216 | consumed tokens: 9718202368 | elapsed time per iteration (s): 141.69 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.242271E-01 | grad norm: 0.380 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default7]: iteration 2318/ 3100 | consumed samples: 4747264 | consumed tokens: 9722396672 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.151877E-01 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.55 | -[default7]: iteration 2319/ 3100 | consumed samples: 4749312 | consumed tokens: 9726590976 | elapsed time per iteration (s): 140.49 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.196396E-01 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.578 | TFLOPs: 148.82 | -[default7]: iteration 2320/ 3100 | consumed samples: 4751360 | consumed tokens: 9730785280 | elapsed time per iteration (s): 141.51 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.243611E-01 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.473 | TFLOPs: 147.75 | -[default7]: iteration 2321/ 3100 | consumed samples: 4753408 | consumed tokens: 9734979584 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.212437E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 2322/ 3100 | consumed samples: 4755456 | consumed tokens: 9739173888 | elapsed time per iteration (s): 141.63 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.145782E-01 | grad norm: 0.369 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.62 | -[default7]: iteration 2323/ 3100 | consumed samples: 4757504 | consumed tokens: 9743368192 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.042384E-01 | grad norm: 0.353 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 2324/ 3100 | consumed samples: 4759552 | consumed tokens: 9747562496 | elapsed time per iteration (s): 141.62 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 6.982729E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.461 | TFLOPs: 147.62 | -[default7]: iteration 2325/ 3100 | consumed samples: 4761600 | consumed tokens: 9751756800 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.092451E-01 | grad norm: 0.341 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.464 | TFLOPs: 147.65 | -[default7]: iteration 2326/ 3100 | consumed samples: 4763648 | consumed tokens: 9755951104 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.150582E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.59 | -[default7]: iteration 2327/ 3100 | consumed samples: 4765696 | consumed tokens: 9760145408 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 6.913494E-01 | grad norm: 0.337 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.51 | -[default7]: iteration 2328/ 3100 | consumed samples: 4767744 | consumed tokens: 9764339712 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 6.957588E-01 | grad norm: 0.350 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.50 | -[default7]: iteration 2329/ 3100 | consumed samples: 4769792 | consumed tokens: 9768534016 | elapsed time per iteration (s): 141.81 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.256360E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.441 | TFLOPs: 147.42 | -[default7]: iteration 2330/ 3100 | consumed samples: 4771840 | consumed tokens: 9772728320 | elapsed time per iteration (s): 141.75 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.121720E-01 | grad norm: 0.358 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.448 | TFLOPs: 147.49 | -[default7]: iteration 2331/ 3100 | consumed samples: 4773888 | consumed tokens: 9776922624 | elapsed time per iteration (s): 141.87 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.259870E-01 | grad norm: 0.353 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.435 | TFLOPs: 147.36 | -[default7]: iteration 2332/ 3100 | consumed samples: 4775936 | consumed tokens: 9781116928 | elapsed time per iteration (s): 141.73 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.055476E-01 | grad norm: 0.347 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.450 | TFLOPs: 147.51 | -[default7]: iteration 2333/ 3100 | consumed samples: 4777984 | consumed tokens: 9785311232 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.258115E-01 | grad norm: 0.348 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.68 | -[default7]: iteration 2334/ 3100 | consumed samples: 4780032 | consumed tokens: 9789505536 | elapsed time per iteration (s): 141.37 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.046494E-01 | grad norm: 0.335 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.486 | TFLOPs: 147.88 | -[default7]: iteration 2335/ 3100 | consumed samples: 4782080 | consumed tokens: 9793699840 | elapsed time per iteration (s): 141.68 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.226317E-01 | grad norm: 0.338 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.455 | TFLOPs: 147.56 | -[default7]: iteration 2336/ 3100 | consumed samples: 4784128 | consumed tokens: 9797894144 | elapsed time per iteration (s): 141.21 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.102132E-01 | grad norm: 0.333 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.503 | TFLOPs: 148.05 | -[default7]: iteration 2337/ 3100 | consumed samples: 4786176 | consumed tokens: 9802088448 | elapsed time per iteration (s): 141.55 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.146807E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.469 | TFLOPs: 147.70 | -[default7]: iteration 2338/ 3100 | consumed samples: 4788224 | consumed tokens: 9806282752 | elapsed time per iteration (s): 141.57 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.089627E-01 | grad norm: 0.340 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.466 | TFLOPs: 147.67 | -[default7]: iteration 2339/ 3100 | consumed samples: 4790272 | consumed tokens: 9810477056 | elapsed time per iteration (s): 141.78 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.121755E-01 | grad norm: 0.338 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.445 | TFLOPs: 147.46 | -[default7]: iteration 2340/ 3100 | consumed samples: 4792320 | consumed tokens: 9814671360 | elapsed time per iteration (s): 141.59 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.076349E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.66 | -[default7]: iteration 2341/ 3100 | consumed samples: 4794368 | consumed tokens: 9818865664 | elapsed time per iteration (s): 141.58 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.098706E-01 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.465 | TFLOPs: 147.67 | -[default7]: iteration 2342/ 3100 | consumed samples: 4796416 | consumed tokens: 9823059968 | elapsed time per iteration (s): 141.64 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.073794E-01 | grad norm: 0.326 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.460 | TFLOPs: 147.61 | -[default7]: iteration 2343/ 3100 | consumed samples: 4798464 | consumed tokens: 9827254272 | elapsed time per iteration (s): 141.74 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.144439E-01 | grad norm: 0.341 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.449 | TFLOPs: 147.50 | -[default7]: iteration 2344/ 3100 | consumed samples: 4800512 | consumed tokens: 9831448576 | elapsed time per iteration (s): 141.56 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.097375E-01 | grad norm: 0.343 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.467 | TFLOPs: 147.68 | -[default7]: iteration 2345/ 3100 | consumed samples: 4802560 | consumed tokens: 9835642880 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 6.992923E-01 | grad norm: 0.334 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.453 | TFLOPs: 147.54 | -[default7]: iteration 2346/ 3100 | consumed samples: 4804608 | consumed tokens: 9839837184 | elapsed time per iteration (s): 139.95 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.203385E-01 | grad norm: 0.363 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.633 | TFLOPs: 149.39 | -[default7]: iteration 2347/ 3100 | consumed samples: 4806656 | consumed tokens: 9844031488 | elapsed time per iteration (s): 140.61 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.223105E-01 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.565 | TFLOPs: 148.69 | -[default7]: iteration 2348/ 3100 | consumed samples: 4808704 | consumed tokens: 9848225792 | elapsed time per iteration (s): 141.45 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.121603E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.479 | TFLOPs: 147.81 | -[default7]: iteration 2349/ 3100 | consumed samples: 4810752 | consumed tokens: 9852420096 | elapsed time per iteration (s): 141.80 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.110015E-01 | grad norm: 0.377 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.443 | TFLOPs: 147.44 | -[default7]: iteration 2350/ 3100 | consumed samples: 4812800 | consumed tokens: 9856614400 | elapsed time per iteration (s): 141.66 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.128224E-01 | grad norm: 0.360 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.457 | TFLOPs: 147.58 | -[default7]: iteration 2351/ 3100 | consumed samples: 4814848 | consumed tokens: 9860808704 | elapsed time per iteration (s): 141.71 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.106890E-01 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.452 | TFLOPs: 147.53 | -[default7]: iteration 2352/ 3100 | consumed samples: 4816896 | consumed tokens: 9865003008 | elapsed time per iteration (s): 141.85 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.091774E-01 | grad norm: 0.375 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.437 | TFLOPs: 147.38 | -[default7]: iteration 2353/ 3100 | consumed samples: 4818944 | consumed tokens: 9869197312 | elapsed time per iteration (s): 141.89 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.193521E-01 | grad norm: 0.356 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.434 | TFLOPs: 147.35 | -[default7]: iteration 2354/ 3100 | consumed samples: 4820992 | consumed tokens: 9873391616 | elapsed time per iteration (s): 141.60 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.090361E-01 | grad norm: 0.353 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.463 | TFLOPs: 147.65 | -[default7]: iteration 2355/ 3100 | consumed samples: 4823040 | consumed tokens: 9877585920 | elapsed time per iteration (s): 140.05 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.038279E-01 | grad norm: 0.341 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.623 | TFLOPs: 149.28 | -[default7]: iteration 2356/ 3100 | consumed samples: 4825088 | consumed tokens: 9881780224 | elapsed time per iteration (s): 141.13 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.208825E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.511 | TFLOPs: 148.14 | -[default4]:[2022-09-11 18:57:15,224] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_67-model_00-model_states.pt... -[default7]: iteration 2357/ 3100 | consumed samples: 4827136 | consumed tokens: 9885974528 | elapsed time per iteration (s): 141.70 | learning rate: 2.000E-05 | global batch size: 2048 | lm loss: 7.001287E-01 | grad norm: 0.384 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 14.454 | TFLOPs: 147.55 | -[default0]:saving checkpoint at iteration 2357 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[2022-09-11 18:57:15,210] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step2357 is begin to save! -[default0]:[2022-09-11 18:57:15,224] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_66-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,588] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_34-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,623] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_41-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,642] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_07-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,641] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_06-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,665] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_20-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,666] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_18-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,588] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_35-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,689] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_53-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,623] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_40-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_14-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_50-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_39-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_37-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_29-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_43-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,665] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_21-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_22-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_68-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_49-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_51-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_54-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_70-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_71-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,665] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_05-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_44-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_52-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_56-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_17-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_64-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_08-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_63-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,712] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_32-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_12-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_55-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_24-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,689] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_42-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_19-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_09-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_11-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_59-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_03-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_15-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_58-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_72-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_38-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,723] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_33-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,693] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_36-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_13-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,665] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_04-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_28-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_48-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,693] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_23-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_27-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_69-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,689] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_16-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_46-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_57-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_60-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_26-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_62-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,723] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_30-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_10-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_47-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,686] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_71_model_states.pt... -[default4]:[2022-09-11 18:57:16,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_71_model_states.pt. -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_25-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_45-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,723] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_31-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:16,695] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_01-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_61-model_00-model_states.pt... -[default4]:[2022-09-11 18:57:16,692] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_65-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:19,913] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_34-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:19,914] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_32_model_states.pt... -[default0]:[2022-09-11 18:57:19,935] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_32_model_states.pt. -[default0]:[2022-09-11 18:57:20,021] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_08-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,021] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_06_model_states.pt... -[default0]:[2022-09-11 18:57:20,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_06_model_states.pt. -[default0]:[2022-09-11 18:57:20,013] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_72-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,013] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_74-model_00-model_states.pt... -[default0]:[2022-09-11 18:57:20,015] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_74-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,015] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_70_model_states.pt... -[default0]:[2022-09-11 18:57:20,017] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_70_model_states.pt. -[default0]:[2022-09-11 18:57:19,978] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_06-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:19,979] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_04_model_states.pt... -[default0]:[2022-09-11 18:57:19,984] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_04_model_states.pt. -[default4]:[2022-09-11 18:57:20,078] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_45-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,079] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_43_model_states.pt... -[default4]:[2022-09-11 18:57:20,082] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_43_model_states.pt. -[default4]:[2022-09-11 18:57:20,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_35-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,016] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_33_model_states.pt... -[default4]:[2022-09-11 18:57:20,019] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_33_model_states.pt. -[default4]:[2022-09-11 18:57:20,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_03-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,090] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_01_model_states.pt... -[default4]:[2022-09-11 18:57:20,093] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_01_model_states.pt. -[default0]:[2022-09-11 18:57:20,150] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_28-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,150] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_26_model_states.pt... -[default0]:[2022-09-11 18:57:20,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_26_model_states.pt. -[default4]:[2022-09-11 18:57:20,117] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_27-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,118] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_25_model_states.pt... -[default4]:[2022-09-11 18:57:20,133] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_25_model_states.pt. -[default0]:[2022-09-11 18:57:20,102] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_40-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,102] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_38_model_states.pt... -[default0]:[2022-09-11 18:57:20,111] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_38_model_states.pt. -[default0]:[2022-09-11 18:57:20,198] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_32-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,199] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_30_model_states.pt... -[default0]:[2022-09-11 18:57:20,202] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_30_model_states.pt. -[default4]:[2022-09-11 18:57:20,244] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_33-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,244] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_31_model_states.pt... -[default4]:[2022-09-11 18:57:20,246] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_31_model_states.pt. -[default4]:[2022-09-11 18:57:20,191] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_07-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,191] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_05_model_states.pt... -[default4]:[2022-09-11 18:57:20,193] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_05_model_states.pt. -[default4]:[2022-09-11 18:57:20,215] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_23-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,215] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_21_model_states.pt... -[default4]:[2022-09-11 18:57:20,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_21_model_states.pt. -[default0]:[2022-09-11 18:57:20,267] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_16-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,267] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_14_model_states.pt... -[default0]:[2022-09-11 18:57:20,270] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_14_model_states.pt. -[default0]:[2022-09-11 18:57:20,228] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_26-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,229] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_24_model_states.pt... -[default0]:[2022-09-11 18:57:20,231] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_24_model_states.pt. -[default0]:[2022-09-11 18:57:20,275] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_18-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,275] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_16_model_states.pt... -[default0]:[2022-09-11 18:57:20,277] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_16_model_states.pt. -[default4]:[2022-09-11 18:57:20,302] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_21-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,302] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_19_model_states.pt... -[default4]:[2022-09-11 18:57:20,304] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_19_model_states.pt. -[default0]:[2022-09-11 18:57:20,316] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_56-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_54_model_states.pt... -[default0]:[2022-09-11 18:57:20,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_54_model_states.pt. -[default0]:[2022-09-11 18:57:20,337] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_42-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,338] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_40_model_states.pt... -[default0]:[2022-09-11 18:57:20,341] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_40_model_states.pt. -[default4]:[2022-09-11 18:57:20,263] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_19-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,263] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_17_model_states.pt... -[default4]:[2022-09-11 18:57:20,266] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_17_model_states.pt. -[default4]:[2022-09-11 18:57:20,303] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_09-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,303] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_07_model_states.pt... -[default4]:[2022-09-11 18:57:20,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_07_model_states.pt. -[default4]:[2022-09-11 18:57:20,281] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_11-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,281] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_09_model_states.pt... -[default4]:[2022-09-11 18:57:20,290] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_09_model_states.pt. -[default0]:[2022-09-11 18:57:20,309] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_46-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,310] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_44_model_states.pt... -[default0]:[2022-09-11 18:57:20,314] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_44_model_states.pt. -[default0]:[2022-09-11 18:57:20,370] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_30-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,371] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_28_model_states.pt... -[default0]:[2022-09-11 18:57:20,378] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_28_model_states.pt. -[default0]:[2022-09-11 18:57:20,356] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_14-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,357] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_12_model_states.pt... -[default0]:[2022-09-11 18:57:20,365] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_12_model_states.pt. -[default4]:[2022-09-11 18:57:20,337] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_29-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,338] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_27_model_states.pt... -[default4]:[2022-09-11 18:57:20,340] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_27_model_states.pt. -[default4]:[2022-09-11 18:57:20,350] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_43-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,350] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_41_model_states.pt... -[default4]:[2022-09-11 18:57:20,353] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_41_model_states.pt. -[default0]:[2022-09-11 18:57:20,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_22-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,319] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_20_model_states.pt... -[default0]:[2022-09-11 18:57:20,322] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_20_model_states.pt. -[default4]:[2022-09-11 18:57:20,378] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_05-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,379] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_03_model_states.pt... -[default4]:[2022-09-11 18:57:20,388] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_03_model_states.pt. -[default0]:[2022-09-11 18:57:20,387] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_44-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,388] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_42_model_states.pt... -[default0]:[2022-09-11 18:57:20,390] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_42_model_states.pt. -[default4]:[2022-09-11 18:57:20,414] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_17-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,414] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_15_model_states.pt... -[default4]:[2022-09-11 18:57:20,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_15_model_states.pt. -[default4]:[2022-09-11 18:57:20,420] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_41-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,420] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_39_model_states.pt... -[default4]:[2022-09-11 18:57:20,425] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_39_model_states.pt. -[default4]:[2022-09-11 18:57:20,384] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_15-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,385] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_13_model_states.pt... -[default4]:[2022-09-11 18:57:20,387] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_13_model_states.pt. -[default0]:[2022-09-11 18:57:20,434] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_38-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,435] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_36_model_states.pt... -[default0]:[2022-09-11 18:57:20,378] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_04-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,379] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_02_model_states.pt... -[default0]:[2022-09-11 18:57:20,388] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_02_model_states.pt. -[default0]:[2022-09-11 18:57:20,448] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_20-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,449] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_18_model_states.pt... -[default0]:[2022-09-11 18:57:20,451] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_18_model_states.pt. -[default0]:[2022-09-11 18:57:20,410] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_10-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,411] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_08_model_states.pt... -[default0]:[2022-09-11 18:57:20,412] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_08_model_states.pt. -[default4]:[2022-09-11 18:57:20,434] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_31-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,435] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_29_model_states.pt... -[default4]:[2022-09-11 18:57:20,436] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_29_model_states.pt. -[default0]:[2022-09-11 18:57:20,486] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_50-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,487] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_48_model_states.pt... -[default0]:[2022-09-11 18:57:20,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_48_model_states.pt. -[default4]:[2022-09-11 18:57:20,476] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_39-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,476] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_37_model_states.pt... -[default4]:[2022-09-11 18:57:20,483] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_37_model_states.pt. -[default0]:[2022-09-11 18:57:20,485] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_68-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,485] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_66_model_states.pt... -[default0]:[2022-09-11 18:57:20,489] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_66_model_states.pt. -[default4]:[2022-09-11 18:57:20,429] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_51-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,429] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_49_model_states.pt... -[default4]:[2022-09-11 18:57:20,433] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_49_model_states.pt. -[default4]:[2022-09-11 18:57:20,524] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_71-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,524] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_69_model_states.pt... -[default4]:[2022-09-11 18:57:20,515] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_63-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_61_model_states.pt... -[default4]:[2022-09-11 18:57:20,517] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_61_model_states.pt. -[default4]:[2022-09-11 18:57:20,485] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_55-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,485] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_53_model_states.pt... -[default4]:[2022-09-11 18:57:20,489] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_53_model_states.pt. -[default0]:[2022-09-11 18:57:20,494] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_24-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,494] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_22_model_states.pt... -[default0]:[2022-09-11 18:57:20,496] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_22_model_states.pt. -[default0]:[2022-09-11 18:57:20,459] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_36_model_states.pt. -[default4]:[2022-09-11 18:57:20,503] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_13-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,503] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_11_model_states.pt... -[default4]:[2022-09-11 18:57:20,505] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_11_model_states.pt. -[default0]:[2022-09-11 18:57:20,558] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_48-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,559] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_46_model_states.pt... -[default0]:[2022-09-11 18:57:20,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_46_model_states.pt. -[default4]:[2022-09-11 18:57:20,491] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_57-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,491] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_55_model_states.pt... -[default4]:[2022-09-11 18:57:20,494] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_55_model_states.pt. -[default0]:[2022-09-11 18:57:20,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_60-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,503] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_58_model_states.pt... -[default0]:[2022-09-11 18:57:20,505] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_58_model_states.pt. -[default4]:[2022-09-11 18:57:20,518] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_47-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,518] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_45_model_states.pt... -[default4]:[2022-09-11 18:57:20,520] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_45_model_states.pt. -[default4]:[2022-09-11 18:57:20,574] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_25-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,575] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_23_model_states.pt... -[default4]:[2022-09-11 18:57:20,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_23_model_states.pt. -[default4]:[2022-09-11 18:57:20,551] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_61-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,551] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_59_model_states.pt... -[default4]:[2022-09-11 18:57:20,556] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_59_model_states.pt. -[default4]:[2022-09-11 18:57:20,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_49-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,546] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_47_model_states.pt... -[default4]:[2022-09-11 18:57:20,549] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_47_model_states.pt. -[default0]:[2022-09-11 18:57:20,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_54-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,541] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_52_model_states.pt... -[default0]:[2022-09-11 18:57:20,543] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_52_model_states.pt. -[default0]:[2022-09-11 18:57:20,614] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_70-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,615] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_68_model_states.pt... -[default4]:[2022-09-11 18:57:20,527] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_69_model_states.pt. -[default0]:[2022-09-11 18:57:20,624] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_52-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,624] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_50_model_states.pt... -[default0]:[2022-09-11 18:57:20,626] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_50_model_states.pt. -[default0]:[2022-09-11 18:57:20,567] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_12-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,568] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_10_model_states.pt... -[default0]:[2022-09-11 18:57:20,569] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_10_model_states.pt. -[default4]:[2022-09-11 18:57:20,614] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_67-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,615] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_65_model_states.pt... -[default4]:[2022-09-11 18:57:20,624] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_65_model_states.pt. -[default0]:[2022-09-11 18:57:20,593] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_36-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,593] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_34_model_states.pt... -[default0]:[2022-09-11 18:57:20,599] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_34_model_states.pt. -[default4]:[2022-09-11 18:57:20,598] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_69-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,598] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_67_model_states.pt... -[default4]:[2022-09-11 18:57:20,601] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_67_model_states.pt. -[default0]:[2022-09-11 18:57:20,634] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_62-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,634] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_60_model_states.pt... -[default0]:[2022-09-11 18:57:20,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_60_model_states.pt. -[default4]:[2022-09-11 18:57:20,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_53-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,685] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_51_model_states.pt... -[default4]:[2022-09-11 18:57:20,687] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_51_model_states.pt. -[default4]:[2022-09-11 18:57:20,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_65-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,670] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_63_model_states.pt... -[default4]:[2022-09-11 18:57:20,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_63_model_states.pt. -[default0]:[2022-09-11 18:57:20,614] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_66-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,615] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_64_model_states.pt... -[default0]:[2022-09-11 18:57:20,625] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_64_model_states.pt. -[default4]:[2022-09-11 18:57:20,621] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_37-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,621] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_35_model_states.pt... -[default4]:[2022-09-11 18:57:20,624] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_35_model_states.pt. -[default0]:[2022-09-11 18:57:20,624] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_68_model_states.pt. -[default0]:[2022-09-11 18:57:20,658] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_64-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,658] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_62_model_states.pt... -[default0]:[2022-09-11 18:57:20,661] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_62_model_states.pt. -[default4]:[2022-09-11 18:57:20,679] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_59-model_00-model_states.pt. -[default4]:[2022-09-11 18:57:20,680] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_57_model_states.pt... -[default4]:[2022-09-11 18:57:20,682] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_57_model_states.pt. -[default0]:[2022-09-11 18:57:20,737] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_58-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:20,737] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_56_model_states.pt... -[default0]:[2022-09-11 18:57:20,739] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_56_model_states.pt. -[default0]:[2022-09-11 18:57:21,381] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/layer_01-model_00-model_states.pt. -[default0]:[2022-09-11 18:57:21,381] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_00_model_states.pt -[default0]:[2022-09-11 18:57:21,381] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_00_model_states.pt... -[default0]:[2022-09-11 18:57:21,383] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/mp_rank_00_model_states.pt. -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt... -[default1]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt... -[default0]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt... -[default6]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt... -[default2]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt... -[default3]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... -[default7]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt... -[default4]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt... -[default5]:[2022-09-11 18:57:21,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt... -[default4]:[2022-09-11 18:57:29,236] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt. -[default4]:[2022-09-11 18:57:29,236] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_41_optim_states.pt -[default3]:[2022-09-11 18:57:29,378] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt. -[default3]:[2022-09-11 18:57:29,378] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_40_optim_states.pt -[default3]:[2022-09-11 18:57:29,707] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt. -[default3]:[2022-09-11 18:57:29,708] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_06_optim_states.pt -[default6]:[2022-09-11 18:57:29,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt. -[default6]:[2022-09-11 18:57:29,932] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_41_optim_states.pt -[default5]:[2022-09-11 18:57:30,331] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt. -[default5]:[2022-09-11 18:57:30,331] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_13_optim_states.pt -[default2]:[2022-09-11 18:57:30,366] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt. -[default2]:[2022-09-11 18:57:30,366] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_60_optim_states.pt -[default2]:[2022-09-11 18:57:30,430] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt. -[default2]:[2022-09-11 18:57:30,430] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_40_optim_states.pt -[default7]:[2022-09-11 18:57:30,428] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt. -[default7]:[2022-09-11 18:57:30,428] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_43_optim_states.pt -[default5]:[2022-09-11 18:57:30,420] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt. -[default5]:[2022-09-11 18:57:30,420] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_41_optim_states.pt -[default7]:[2022-09-11 18:57:30,463] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt. -[default7]:[2022-09-11 18:57:30,463] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_41_optim_states.pt -[default1]:[2022-09-11 18:57:30,578] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt. -[default1]:[2022-09-11 18:57:30,578] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_48_optim_states.pt -[default0]:[2022-09-11 18:57:30,629] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt. -[default0]:[2022-09-11 18:57:30,629] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_30_optim_states.pt -[default0]:[2022-09-11 18:57:30,644] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt. -[default0]:[2022-09-11 18:57:30,644] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_36_optim_states.pt -[default7]:[2022-09-11 18:57:30,683] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt. -[default7]:[2022-09-11 18:57:30,684] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_15_optim_states.pt -[default1]:[2022-09-11 18:57:30,710] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt. -[default1]:[2022-09-11 18:57:30,710] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_40_optim_states.pt -[default6]:[2022-09-11 18:57:30,759] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt. -[default6]:[2022-09-11 18:57:30,759] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_13_optim_states.pt -[default3]:[2022-09-11 18:57:30,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt. -[default3]:[2022-09-11 18:57:30,783] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_18_optim_states.pt -[default0]:[2022-09-11 18:57:30,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt. -[default0]:[2022-09-11 18:57:30,837] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_40_optim_states.pt -[default0]:[2022-09-11 18:57:30,986] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt. -[default0]:[2022-09-11 18:57:30,986] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_42_optim_states.pt -[default0]:[2022-09-11 18:57:31,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt. -[default0]:[2022-09-11 18:57:31,025] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_58_optim_states.pt -[default4]:[2022-09-11 18:57:31,090] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt. -[default4]:[2022-09-11 18:57:31,090] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_69_optim_states.pt -[default3]:[2022-09-11 18:57:31,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt. -[default3]:[2022-09-11 18:57:31,070] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_52_optim_states.pt -[default5]:[2022-09-11 18:57:31,226] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt. -[default5]:[2022-09-11 18:57:31,226] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_05_optim_states.pt -[default3]:[2022-09-11 18:57:31,303] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt. -[default3]:[2022-09-11 18:57:31,303] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_20_optim_states.pt -[default1]:[2022-09-11 18:57:31,328] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt. -[default1]:[2022-09-11 18:57:31,328] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_42_optim_states.pt -[default2]:[2022-09-11 18:57:31,270] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt. -[default2]:[2022-09-11 18:57:31,270] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_24_optim_states.pt -[default7]:[2022-09-11 18:57:31,426] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt. -[default7]:[2022-09-11 18:57:31,427] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_13_optim_states.pt -[default0]:[2022-09-11 18:57:31,430] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt. -[default0]:[2022-09-11 18:57:31,430] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_14_optim_states.pt -[default3]:[2022-09-11 18:57:31,470] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt. -[default3]:[2022-09-11 18:57:31,470] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_12_optim_states.pt -[default2]:[2022-09-11 18:57:31,444] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt. -[default2]:[2022-09-11 18:57:31,444] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_12_optim_states.pt -[default0]:[2022-09-11 18:57:31,418] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt. -[default0]:[2022-09-11 18:57:31,419] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_12_optim_states.pt -[default2]:[2022-09-11 18:57:31,477] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt. -[default2]:[2022-09-11 18:57:31,477] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_42_optim_states.pt -[default4]:[2022-09-11 18:57:31,424] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt. -[default4]:[2022-09-11 18:57:31,424] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_35_optim_states.pt -[default5]:[2022-09-11 18:57:31,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt. -[default5]:[2022-09-11 18:57:31,503] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_43_optim_states.pt -[default3]:[2022-09-11 18:57:31,538] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt. -[default3]:[2022-09-11 18:57:31,538] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_48_optim_states.pt -[default1]:[2022-09-11 18:57:31,573] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt. -[default1]:[2022-09-11 18:57:31,573] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_12_optim_states.pt -[default4]:[2022-09-11 18:57:31,512] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt. -[default4]:[2022-09-11 18:57:31,512] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_43_optim_states.pt -[default3]:[2022-09-11 18:57:31,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt. -[default3]:[2022-09-11 18:57:31,547] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_68_optim_states.pt -[default4]:[2022-09-11 18:57:31,621] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt. -[default4]:[2022-09-11 18:57:31,621] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_49_optim_states.pt -[default1]:[2022-09-11 18:57:31,580] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt. -[default1]:[2022-09-11 18:57:31,580] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_14_optim_states.pt -[default6]:[2022-09-11 18:57:31,648] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt. -[default6]:[2022-09-11 18:57:31,648] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_65_optim_states.pt -[default4]:[2022-09-11 18:57:31,656] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt. -[default4]:[2022-09-11 18:57:31,656] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_13_optim_states.pt -[default1]:[2022-09-11 18:57:31,686] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt. -[default1]:[2022-09-11 18:57:31,686] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_68_optim_states.pt -[default1]:[2022-09-11 18:57:31,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt. -[default1]:[2022-09-11 18:57:31,615] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_04_optim_states.pt -[default7]:[2022-09-11 18:57:31,639] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt. -[default7]:[2022-09-11 18:57:31,639] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_09_optim_states.pt -[default6]:[2022-09-11 18:57:31,669] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt. -[default6]:[2022-09-11 18:57:31,669] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_35_optim_states.pt -[default0]:[2022-09-11 18:57:31,733] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt. -[default0]:[2022-09-11 18:57:31,733] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_62_optim_states.pt -[default3]:[2022-09-11 18:57:31,684] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt. -[default3]:[2022-09-11 18:57:31,684] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_30_optim_states.pt -[default2]:[2022-09-11 18:57:31,785] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt. -[default2]:[2022-09-11 18:57:31,785] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_66_optim_states.pt -[default5]:[2022-09-11 18:57:31,738] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt. -[default5]:[2022-09-11 18:57:31,738] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_57_optim_states.pt -[default7]:[2022-09-11 18:57:31,791] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt. -[default7]:[2022-09-11 18:57:31,791] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_35_optim_states.pt -[default0]:[2022-09-11 18:57:31,945] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt. -[default0]:[2022-09-11 18:57:31,945] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_48_optim_states.pt -[default5]:[2022-09-11 18:57:31,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt. -[default5]:[2022-09-11 18:57:31,994] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_55_optim_states.pt -[default7]:[2022-09-11 18:57:32,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt. -[default7]:[2022-09-11 18:57:32,052] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_49_optim_states.pt -[default3]:[2022-09-11 18:57:32,106] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt. -[default3]:[2022-09-11 18:57:32,106] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_32_optim_states.pt -[default2]:[2022-09-11 18:57:32,076] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt. -[default2]:[2022-09-11 18:57:32,076] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_48_optim_states.pt -[default5]:[2022-09-11 18:57:32,142] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt. -[default5]:[2022-09-11 18:57:32,142] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_49_optim_states.pt -[default0]:[2022-09-11 18:57:32,068] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt. -[default0]:[2022-09-11 18:57:32,068] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_32_optim_states.pt -[default2]:[2022-09-11 18:57:32,185] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt. -[default2]:[2022-09-11 18:57:32,185] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_34_optim_states.pt -[default5]:[2022-09-11 18:57:32,208] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt. -[default5]:[2022-09-11 18:57:32,208] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_69_optim_states.pt -[default0]:[2022-09-11 18:57:32,162] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt. -[default0]:[2022-09-11 18:57:32,163] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_06_optim_states.pt -[default6]:[2022-09-11 18:57:32,234] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt. -[default6]:[2022-09-11 18:57:32,234] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_43_optim_states.pt -[default7]:[2022-09-11 18:57:32,236] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt. -[default7]:[2022-09-11 18:57:32,236] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_53_optim_states.pt -[default7]:[2022-09-11 18:57:32,300] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt. -[default7]:[2022-09-11 18:57:32,300] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_21_optim_states.pt -[default6]:[2022-09-11 18:57:32,271] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt. -[default6]:[2022-09-11 18:57:32,271] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_49_optim_states.pt -[default4]:[2022-09-11 18:57:32,312] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt. -[default4]:[2022-09-11 18:57:32,312] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_09_optim_states.pt -[default3]:[2022-09-11 18:57:32,315] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt. -[default3]:[2022-09-11 18:57:32,315] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_42_optim_states.pt -[default0]:[2022-09-11 18:57:32,406] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt. -[default0]:[2022-09-11 18:57:32,406] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_52_optim_states.pt -[default0]:[2022-09-11 18:57:32,372] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt. -[default0]:[2022-09-11 18:57:32,372] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_68_optim_states.pt -[default2]:[2022-09-11 18:57:32,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt. -[default2]:[2022-09-11 18:57:32,416] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_64_optim_states.pt -[default3]:[2022-09-11 18:57:32,393] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt. -[default3]:[2022-09-11 18:57:32,394] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_34_optim_states.pt -[default2]:[2022-09-11 18:57:32,422] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt. -[default2]:[2022-09-11 18:57:32,423] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_26_optim_states.pt -[default0]:[2022-09-11 18:57:32,460] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt. -[default0]:[2022-09-11 18:57:32,460] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_44_optim_states.pt -[default3]:[2022-09-11 18:57:32,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt. -[default3]:[2022-09-11 18:57:32,503] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_54_optim_states.pt -[default4]:[2022-09-11 18:57:32,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt. -[default4]:[2022-09-11 18:57:32,540] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_15_optim_states.pt -[default7]:[2022-09-11 18:57:32,507] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt. -[default7]:[2022-09-11 18:57:32,507] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_57_optim_states.pt -[default3]:[2022-09-11 18:57:32,551] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt. -[default3]:[2022-09-11 18:57:32,551] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_14_optim_states.pt -[default4]:[2022-09-11 18:57:32,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt. -[default4]:[2022-09-11 18:57:32,488] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_21_optim_states.pt -[default4]:[2022-09-11 18:57:32,503] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt. -[default4]:[2022-09-11 18:57:32,503] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_25_optim_states.pt -[default1]:[2022-09-11 18:57:32,494] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt. -[default1]:[2022-09-11 18:57:32,494] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_64_optim_states.pt -[default7]:[2022-09-11 18:57:32,570] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt. -[default7]:[2022-09-11 18:57:32,570] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_69_optim_states.pt -[default2]:[2022-09-11 18:57:32,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt. -[default2]:[2022-09-11 18:57:32,647] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_68_optim_states.pt -[default5]:[2022-09-11 18:57:32,594] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt. -[default5]:[2022-09-11 18:57:32,595] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_25_optim_states.pt -[default1]:[2022-09-11 18:57:32,690] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt. -[default1]:[2022-09-11 18:57:32,690] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_20_optim_states.pt -[default6]:[2022-09-11 18:57:32,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt. -[default6]:[2022-09-11 18:57:32,672] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_69_optim_states.pt -[default1]:[2022-09-11 18:57:32,677] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt. -[default1]:[2022-09-11 18:57:32,677] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_34_optim_states.pt -[default3]:[2022-09-11 18:57:32,740] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt. -[default3]:[2022-09-11 18:57:32,740] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_36_optim_states.pt -[default1]:[2022-09-11 18:57:32,803] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt. -[default1]:[2022-09-11 18:57:32,803] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_58_optim_states.pt -[default4]:[2022-09-11 18:57:32,807] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt. -[default4]:[2022-09-11 18:57:32,807] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_61_optim_states.pt -[default0]:[2022-09-11 18:57:32,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt. -[default0]:[2022-09-11 18:57:32,870] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_34_optim_states.pt -[default0]:[2022-09-11 18:57:32,858] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt. -[default0]:[2022-09-11 18:57:32,858] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_04_optim_states.pt -[default2]:[2022-09-11 18:57:32,838] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt. -[default2]:[2022-09-11 18:57:32,838] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_14_optim_states.pt -[default7]:[2022-09-11 18:57:32,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt. -[default7]:[2022-09-11 18:57:32,856] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_31_optim_states.pt -[default0]:[2022-09-11 18:57:32,904] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt. -[default0]:[2022-09-11 18:57:32,904] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_20_optim_states.pt -[default7]:[2022-09-11 18:57:32,898] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt. -[default7]:[2022-09-11 18:57:32,899] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_29_optim_states.pt -[default7]:[2022-09-11 18:57:32,937] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt. -[default7]:[2022-09-11 18:57:32,937] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_23_optim_states.pt -[default5]:[2022-09-11 18:57:32,943] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt. -[default5]:[2022-09-11 18:57:32,943] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_61_optim_states.pt -[default4]:[2022-09-11 18:57:32,926] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt. -[default4]:[2022-09-11 18:57:32,926] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_07_optim_states.pt -[default6]:[2022-09-11 18:57:32,921] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt. -[default6]:[2022-09-11 18:57:32,921] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_53_optim_states.pt -[default3]:[2022-09-11 18:57:32,903] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt. -[default3]:[2022-09-11 18:57:32,903] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_58_optim_states.pt -[default0]:[2022-09-11 18:57:32,899] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt. -[default0]:[2022-09-11 18:57:32,899] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_08_optim_states.pt -[default2]:[2022-09-11 18:57:32,927] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt. -[default2]:[2022-09-11 18:57:32,927] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_20_optim_states.pt -[default2]:[2022-09-11 18:57:32,995] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt. -[default2]:[2022-09-11 18:57:32,995] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_30_optim_states.pt -[default5]:[2022-09-11 18:57:32,938] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt. -[default5]:[2022-09-11 18:57:32,938] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_15_optim_states.pt -[default3]:[2022-09-11 18:57:33,002] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt. -[default3]:[2022-09-11 18:57:33,002] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_22_optim_states.pt -[default2]:[2022-09-11 18:57:33,001] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt. -[default2]:[2022-09-11 18:57:33,001] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_54_optim_states.pt -[default7]:[2022-09-11 18:57:32,940] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt. -[default7]:[2022-09-11 18:57:32,940] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_05_optim_states.pt -[default2]:[2022-09-11 18:57:32,983] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt. -[default2]:[2022-09-11 18:57:32,983] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_52_optim_states.pt -[default4]:[2022-09-11 18:57:32,958] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt. -[default4]:[2022-09-11 18:57:32,959] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_53_optim_states.pt -[default7]:[2022-09-11 18:57:33,019] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt. -[default7]:[2022-09-11 18:57:33,019] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_07_optim_states.pt -[default2]:[2022-09-11 18:57:32,974] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt. -[default2]:[2022-09-11 18:57:32,974] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_22_optim_states.pt -[default7]:[2022-09-11 18:57:33,010] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt. -[default7]:[2022-09-11 18:57:33,011] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_55_optim_states.pt -[default1]:[2022-09-11 18:57:33,027] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt. -[default1]:[2022-09-11 18:57:33,028] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_06_optim_states.pt -[default7]:[2022-09-11 18:57:33,064] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt. -[default7]:[2022-09-11 18:57:33,064] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_25_optim_states.pt -[default3]:[2022-09-11 18:57:33,090] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt. -[default3]:[2022-09-11 18:57:33,090] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_04_optim_states.pt -[default6]:[2022-09-11 18:57:33,045] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt. -[default6]:[2022-09-11 18:57:33,045] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_15_optim_states.pt -[default1]:[2022-09-11 18:57:33,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt. -[default1]:[2022-09-11 18:57:33,072] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_24_optim_states.pt -[default6]:[2022-09-11 18:57:33,068] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt. -[default6]:[2022-09-11 18:57:33,069] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_21_optim_states.pt -[default0]:[2022-09-11 18:57:33,121] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt. -[default0]:[2022-09-11 18:57:33,121] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_66_optim_states.pt -[default1]:[2022-09-11 18:57:33,058] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt. -[default1]:[2022-09-11 18:57:33,058] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_60_optim_states.pt -[default1]:[2022-09-11 18:57:33,134] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt. -[default1]:[2022-09-11 18:57:33,135] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_52_optim_states.pt -[default3]:[2022-09-11 18:57:33,135] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt. -[default3]:[2022-09-11 18:57:33,135] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_60_optim_states.pt -[default6]:[2022-09-11 18:57:33,148] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt. -[default6]:[2022-09-11 18:57:33,148] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_31_optim_states.pt -[default6]:[2022-09-11 18:57:33,138] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt. -[default6]:[2022-09-11 18:57:33,139] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_61_optim_states.pt -[default6]:[2022-09-11 18:57:33,121] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt. -[default6]:[2022-09-11 18:57:33,121] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_19_optim_states.pt -[default2]:[2022-09-11 18:57:33,138] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt. -[default2]:[2022-09-11 18:57:33,138] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_06_optim_states.pt -[default2]:[2022-09-11 18:57:33,111] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt. -[default2]:[2022-09-11 18:57:33,111] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_36_optim_states.pt -[default6]:[2022-09-11 18:57:33,144] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt. -[default6]:[2022-09-11 18:57:33,145] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_07_optim_states.pt -[default0]:[2022-09-11 18:57:33,102] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt. -[default0]:[2022-09-11 18:57:33,102] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_24_optim_states.pt -[default6]:[2022-09-11 18:57:33,215] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt. -[default6]:[2022-09-11 18:57:33,215] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_55_optim_states.pt -[default6]:[2022-09-11 18:57:33,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt. -[default6]:[2022-09-11 18:57:33,203] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_25_optim_states.pt -[default4]:[2022-09-11 18:57:33,205] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt. -[default4]:[2022-09-11 18:57:33,205] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_31_optim_states.pt -[default3]:[2022-09-11 18:57:33,253] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt. -[default3]:[2022-09-11 18:57:33,253] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_24_optim_states.pt -[default4]:[2022-09-11 18:57:33,245] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt. -[default4]:[2022-09-11 18:57:33,246] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_05_optim_states.pt -[default5]:[2022-09-11 18:57:33,255] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt. -[default5]:[2022-09-11 18:57:33,255] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_21_optim_states.pt -[default5]:[2022-09-11 18:57:33,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt. -[default5]:[2022-09-11 18:57:33,310] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_31_optim_states.pt -[default1]:[2022-09-11 18:57:33,301] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt. -[default1]:[2022-09-11 18:57:33,302] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_18_optim_states.pt -[default3]:[2022-09-11 18:57:33,293] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt. -[default3]:[2022-09-11 18:57:33,293] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_62_optim_states.pt -[default6]:[2022-09-11 18:57:33,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt. -[default6]:[2022-09-11 18:57:33,313] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_05_optim_states.pt -[default2]:[2022-09-11 18:57:33,344] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt. -[default2]:[2022-09-11 18:57:33,344] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_08_optim_states.pt -[default0]:[2022-09-11 18:57:33,347] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt. -[default0]:[2022-09-11 18:57:33,347] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_54_optim_states.pt -[default0]:[2022-09-11 18:57:33,328] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt. -[default0]:[2022-09-11 18:57:33,328] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_22_optim_states.pt -[default4]:[2022-09-11 18:57:33,336] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt. -[default4]:[2022-09-11 18:57:33,336] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_65_optim_states.pt -[default2]:[2022-09-11 18:57:33,348] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt. -[default2]:[2022-09-11 18:57:33,348] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_58_optim_states.pt -[default1]:[2022-09-11 18:57:33,298] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt. -[default1]:[2022-09-11 18:57:33,298] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_54_optim_states.pt -[default5]:[2022-09-11 18:57:33,331] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt. -[default5]:[2022-09-11 18:57:33,331] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_09_optim_states.pt -[default5]:[2022-09-11 18:57:33,356] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt. -[default5]:[2022-09-11 18:57:33,357] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_53_optim_states.pt -[default4]:[2022-09-11 18:57:33,348] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt. -[default4]:[2022-09-11 18:57:33,349] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_29_optim_states.pt -[default2]:[2022-09-11 18:57:33,322] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt. -[default2]:[2022-09-11 18:57:33,322] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_04_optim_states.pt -[default4]:[2022-09-11 18:57:33,411] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt. -[default4]:[2022-09-11 18:57:33,412] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_19_optim_states.pt -[default1]:[2022-09-11 18:57:33,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt. -[default1]:[2022-09-11 18:57:33,385] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_44_optim_states.pt -[default7]:[2022-09-11 18:57:33,442] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt. -[default7]:[2022-09-11 18:57:33,442] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_67_optim_states.pt -[default4]:[2022-09-11 18:57:33,413] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt. -[default4]:[2022-09-11 18:57:33,413] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_23_optim_states.pt -[default4]:[2022-09-11 18:57:33,457] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt. -[default4]:[2022-09-11 18:57:33,457] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_59_optim_states.pt -[default7]:[2022-09-11 18:57:33,499] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt. -[default7]:[2022-09-11 18:57:33,499] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_61_optim_states.pt -[default3]:[2022-09-11 18:57:33,458] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt. -[default3]:[2022-09-11 18:57:33,458] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_08_optim_states.pt -[default4]:[2022-09-11 18:57:33,517] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt. -[default4]:[2022-09-11 18:57:33,517] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_37_optim_states.pt -[default6]:[2022-09-11 18:57:33,514] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt. -[default6]:[2022-09-11 18:57:33,514] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_09_optim_states.pt -[default1]:[2022-09-11 18:57:33,494] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt. -[default1]:[2022-09-11 18:57:33,494] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_28_optim_states.pt -[default3]:[2022-09-11 18:57:33,533] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt. -[default3]:[2022-09-11 18:57:33,534] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_64_optim_states.pt -[default0]:[2022-09-11 18:57:33,523] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt. -[default0]:[2022-09-11 18:57:33,524] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_26_optim_states.pt -[default4]:[2022-09-11 18:57:33,568] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt. -[default4]:[2022-09-11 18:57:33,568] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_45_optim_states.pt -[default5]:[2022-09-11 18:57:33,584] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt. -[default5]:[2022-09-11 18:57:33,584] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_19_optim_states.pt -[default5]:[2022-09-11 18:57:33,553] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt. -[default5]:[2022-09-11 18:57:33,553] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_35_optim_states.pt -[default5]:[2022-09-11 18:57:33,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt. -[default5]:[2022-09-11 18:57:33,615] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_23_optim_states.pt -[default0]:[2022-09-11 18:57:33,598] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt. -[default0]:[2022-09-11 18:57:33,598] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_56_optim_states.pt -[default5]:[2022-09-11 18:57:33,619] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt. -[default5]:[2022-09-11 18:57:33,619] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_27_optim_states.pt -[default6]:[2022-09-11 18:57:33,678] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt. -[default6]:[2022-09-11 18:57:33,678] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_45_optim_states.pt -[default1]:[2022-09-11 18:57:33,626] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt. -[default1]:[2022-09-11 18:57:33,626] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_22_optim_states.pt -[default5]:[2022-09-11 18:57:33,604] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt. -[default5]:[2022-09-11 18:57:33,605] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_59_optim_states.pt -[default7]:[2022-09-11 18:57:33,627] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt. -[default7]:[2022-09-11 18:57:33,627] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_65_optim_states.pt -[default7]:[2022-09-11 18:57:33,602] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt. -[default7]:[2022-09-11 18:57:33,603] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_19_optim_states.pt -[default6]:[2022-09-11 18:57:33,627] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt. -[default6]:[2022-09-11 18:57:33,628] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_23_optim_states.pt -[default3]:[2022-09-11 18:57:33,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt. -[default3]:[2022-09-11 18:57:33,675] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_56_optim_states.pt -[default1]:[2022-09-11 18:57:33,724] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt. -[default1]:[2022-09-11 18:57:33,724] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_66_optim_states.pt -[default7]:[2022-09-11 18:57:33,677] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt. -[default7]:[2022-09-11 18:57:33,677] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_59_optim_states.pt -[default6]:[2022-09-11 18:57:33,732] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt. -[default6]:[2022-09-11 18:57:33,732] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_67_optim_states.pt -[default2]:[2022-09-11 18:57:33,651] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt. -[default2]:[2022-09-11 18:57:33,651] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_18_optim_states.pt -[default3]:[2022-09-11 18:57:33,668] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt. -[default3]:[2022-09-11 18:57:33,669] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_26_optim_states.pt -[default3]:[2022-09-11 18:57:33,678] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt. -[default3]:[2022-09-11 18:57:33,678] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_66_optim_states.pt -[default5]:[2022-09-11 18:57:33,742] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt. -[default5]:[2022-09-11 18:57:33,742] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_65_optim_states.pt -[default0]:[2022-09-11 18:57:33,713] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt. -[default0]:[2022-09-11 18:57:33,713] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_18_optim_states.pt -[default4]:[2022-09-11 18:57:33,781] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt. -[default4]:[2022-09-11 18:57:33,781] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_63_optim_states.pt -[default6]:[2022-09-11 18:57:33,750] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt. -[default6]:[2022-09-11 18:57:33,750] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_37_optim_states.pt -[default5]:[2022-09-11 18:57:33,810] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt. -[default5]:[2022-09-11 18:57:33,810] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_67_optim_states.pt -[default5]:[2022-09-11 18:57:33,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt. -[default5]:[2022-09-11 18:57:33,808] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_63_optim_states.pt -[default5]:[2022-09-11 18:57:33,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt. -[default5]:[2022-09-11 18:57:33,770] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_37_optim_states.pt -[default1]:[2022-09-11 18:57:33,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt. -[default1]:[2022-09-11 18:57:33,776] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_62_optim_states.pt -[default6]:[2022-09-11 18:57:33,792] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt. -[default6]:[2022-09-11 18:57:33,792] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_27_optim_states.pt -[default4]:[2022-09-11 18:57:33,867] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt. -[default4]:[2022-09-11 18:57:33,867] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_67_optim_states.pt -[default5]:[2022-09-11 18:57:33,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt. -[default5]:[2022-09-11 18:57:33,854] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_07_optim_states.pt -[default6]:[2022-09-11 18:57:33,816] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt. -[default6]:[2022-09-11 18:57:33,817] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_59_optim_states.pt -[default7]:[2022-09-11 18:57:33,824] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt. -[default7]:[2022-09-11 18:57:33,824] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_37_optim_states.pt -[default7]:[2022-09-11 18:57:33,851] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt. -[default7]:[2022-09-11 18:57:33,851] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_27_optim_states.pt -[default0]:[2022-09-11 18:57:33,852] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt. -[default0]:[2022-09-11 18:57:33,852] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_64_optim_states.pt -[default4]:[2022-09-11 18:57:33,911] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt. -[default4]:[2022-09-11 18:57:33,911] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_27_optim_states.pt -[default1]:[2022-09-11 18:57:33,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt. -[default1]:[2022-09-11 18:57:33,856] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_36_optim_states.pt -[default4]:[2022-09-11 18:57:33,904] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt. -[default4]:[2022-09-11 18:57:33,905] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_55_optim_states.pt -[default4]:[2022-09-11 18:57:33,909] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt. -[default4]:[2022-09-11 18:57:33,909] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_33_optim_states.pt -[default6]:[2022-09-11 18:57:33,988] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt. -[default6]:[2022-09-11 18:57:33,989] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_63_optim_states.pt -[default2]:[2022-09-11 18:57:33,937] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt. -[default2]:[2022-09-11 18:57:33,937] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_56_optim_states.pt -[default2]:[2022-09-11 18:57:33,939] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt. -[default2]:[2022-09-11 18:57:33,939] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_44_optim_states.pt -[default1]:[2022-09-11 18:57:34,034] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt. -[default1]:[2022-09-11 18:57:34,034] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_26_optim_states.pt -[default7]:[2022-09-11 18:57:33,956] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt. -[default7]:[2022-09-11 18:57:33,956] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_63_optim_states.pt -[default5]:[2022-09-11 18:57:34,001] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt. -[default5]:[2022-09-11 18:57:34,001] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_45_optim_states.pt -[default3]:[2022-09-11 18:57:34,048] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt. -[default3]:[2022-09-11 18:57:34,048] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_44_optim_states.pt -[default2]:[2022-09-11 18:57:34,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt. -[default2]:[2022-09-11 18:57:34,012] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_28_optim_states.pt -[default6]:[2022-09-11 18:57:34,081] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt. -[default6]:[2022-09-11 18:57:34,081] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_57_optim_states.pt -[default7]:[2022-09-11 18:57:34,071] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt. -[default7]:[2022-09-11 18:57:34,071] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_03_optim_states.pt -[default1]:[2022-09-11 18:57:34,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt. -[default1]:[2022-09-11 18:57:34,056] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_32_optim_states.pt -[default2]:[2022-09-11 18:57:34,080] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt. -[default2]:[2022-09-11 18:57:34,080] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_32_optim_states.pt -[default3]:[2022-09-11 18:57:34,227] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt. -[default3]:[2022-09-11 18:57:34,227] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_28_optim_states.pt -[default3]:[2022-09-11 18:57:34,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt. -[default3]:[2022-09-11 18:57:34,203] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_02_optim_states.pt -[default1]:[2022-09-11 18:57:34,235] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt. -[default1]:[2022-09-11 18:57:34,235] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_08_optim_states.pt -[default5]:[2022-09-11 18:57:34,298] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt. -[default5]:[2022-09-11 18:57:34,299] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_29_optim_states.pt -[default2]:[2022-09-11 18:57:34,372] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt. -[default2]:[2022-09-11 18:57:34,372] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_10_optim_states.pt -[default0]:[2022-09-11 18:57:34,483] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt. -[default0]:[2022-09-11 18:57:34,484] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_28_optim_states.pt -[default6]:[2022-09-11 18:57:34,472] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt. -[default6]:[2022-09-11 18:57:34,473] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_29_optim_states.pt -[default7]:[2022-09-11 18:57:34,599] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt. -[default7]:[2022-09-11 18:57:34,599] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_33_optim_states.pt -[default4]:[2022-09-11 18:57:34,621] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt. -[default4]:[2022-09-11 18:57:34,621] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_11_optim_states.pt -[default7]:[2022-09-11 18:57:34,774] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt. -[default7]:[2022-09-11 18:57:34,774] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_45_optim_states.pt -[default4]:[2022-09-11 18:57:34,802] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt. -[default4]:[2022-09-11 18:57:34,802] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_03_optim_states.pt -[default6]:[2022-09-11 18:57:34,794] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt. -[default6]:[2022-09-11 18:57:34,794] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_03_optim_states.pt -[default2]:[2022-09-11 18:57:34,924] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt. -[default2]:[2022-09-11 18:57:34,925] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_46_optim_states.pt -[default1]:[2022-09-11 18:57:34,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt. -[default1]:[2022-09-11 18:57:34,887] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_10_optim_states.pt -[default1]:[2022-09-11 18:57:35,009] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt. -[default1]:[2022-09-11 18:57:35,009] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_56_optim_states.pt -[default1]:[2022-09-11 18:57:35,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt. -[default1]:[2022-09-11 18:57:35,025] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_02_optim_states.pt -[default2]:[2022-09-11 18:57:35,145] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt. -[default2]:[2022-09-11 18:57:35,145] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_02_optim_states.pt -[default6]:[2022-09-11 18:57:35,154] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt. -[default6]:[2022-09-11 18:57:35,155] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_33_optim_states.pt -[default4]:[2022-09-11 18:57:35,253] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt. -[default4]:[2022-09-11 18:57:35,253] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_57_optim_states.pt -[default1]:[2022-09-11 18:57:35,447] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt. -[default1]:[2022-09-11 18:57:35,447] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_30_optim_states.pt -[default2]:[2022-09-11 18:57:35,444] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt. -[default2]:[2022-09-11 18:57:35,444] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_62_optim_states.pt -[default0]:[2022-09-11 18:57:35,522] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt. -[default0]:[2022-09-11 18:57:35,523] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_02_optim_states.pt -[default5]:[2022-09-11 18:57:35,640] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt. -[default5]:[2022-09-11 18:57:35,641] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_11_optim_states.pt -[default0]:[2022-09-11 18:57:36,030] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt. -[default0]:[2022-09-11 18:57:36,030] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_46_optim_states.pt -[default1]:[2022-09-11 18:57:36,066] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt. -[default1]:[2022-09-11 18:57:36,066] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_46_optim_states.pt -[default3]:[2022-09-11 18:57:36,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt. -[default3]:[2022-09-11 18:57:36,079] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_10_optim_states.pt -[default3]:[2022-09-11 18:57:36,278] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt. -[default3]:[2022-09-11 18:57:36,278] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_46_optim_states.pt -[default4]:[2022-09-11 18:57:36,308] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt. -[default4]:[2022-09-11 18:57:36,308] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_47_optim_states.pt -[default5]:[2022-09-11 18:57:36,339] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt. -[default5]:[2022-09-11 18:57:36,339] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_47_optim_states.pt -[default0]:[2022-09-11 18:57:36,330] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt. -[default0]:[2022-09-11 18:57:36,331] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_10_optim_states.pt -[default5]:[2022-09-11 18:57:36,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt. -[default5]:[2022-09-11 18:57:36,335] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_03_optim_states.pt -[default5]:[2022-09-11 18:57:36,389] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt. -[default5]:[2022-09-11 18:57:36,389] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_33_optim_states.pt -[default0]:[2022-09-11 18:57:36,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt. -[default0]:[2022-09-11 18:57:36,417] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_60_optim_states.pt -[default6]:[2022-09-11 18:57:36,520] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt. -[default6]:[2022-09-11 18:57:36,520] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_39_optim_states.pt -[default6]:[2022-09-11 18:57:36,869] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt. -[default6]:[2022-09-11 18:57:36,869] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_11_optim_states.pt -[default7]:[2022-09-11 18:57:36,922] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. -[default7]:[2022-09-11 18:57:36,922] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt -[default5]:[2022-09-11 18:57:37,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt. -[default5]:[2022-09-11 18:57:37,013] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_39_optim_states.pt -[default5]:[2022-09-11 18:57:37,289] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. -[default5]:[2022-09-11 18:57:37,289] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt -[default7]:[2022-09-11 18:57:37,422] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt. -[default7]:[2022-09-11 18:57:37,422] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_11_optim_states.pt -[default6]:[2022-09-11 18:57:37,842] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. -[default6]:[2022-09-11 18:57:37,842] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt -[default4]:[2022-09-11 18:57:37,959] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt. -[default4]:[2022-09-11 18:57:37,959] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_39_optim_states.pt -[default4]:[2022-09-11 18:57:38,022] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. -[default4]:[2022-09-11 18:57:38,023] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt -[default2]:[2022-09-11 18:57:38,098] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt. -[default2]:[2022-09-11 18:57:38,099] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_38_optim_states.pt -[default7]:[2022-09-11 18:57:38,211] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt. -[default7]:[2022-09-11 18:57:38,212] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_39_optim_states.pt -[default2]:[2022-09-11 18:57:39,141] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt. -[default2]:[2022-09-11 18:57:39,141] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_16_optim_states.pt -[default3]:[2022-09-11 18:57:40,195] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt. -[default3]:[2022-09-11 18:57:40,196] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_38_optim_states.pt -[default1]:[2022-09-11 18:57:41,657] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt. -[default1]:[2022-09-11 18:57:41,658] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_16_optim_states.pt -[default0]:[2022-09-11 18:57:41,686] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt. -[default0]:[2022-09-11 18:57:41,686] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_16_optim_states.pt -[default7]:[2022-09-11 18:57:42,087] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt. -[default7]:[2022-09-11 18:57:42,087] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_47_optim_states.pt -[default3]:[2022-09-11 18:57:42,095] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt. -[default3]:[2022-09-11 18:57:42,096] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_16_optim_states.pt -[default6]:[2022-09-11 18:57:42,155] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt. -[default6]:[2022-09-11 18:57:42,155] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_47_optim_states.pt -[default4]:[2022-09-11 18:57:42,200] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt. -[default4]:[2022-09-11 18:57:42,201] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_17_optim_states.pt -[default5]:[2022-09-11 18:57:42,252] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt. -[default5]:[2022-09-11 18:57:42,252] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_17_optim_states.pt -[default1]:[2022-09-11 18:57:42,970] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt. -[default1]:[2022-09-11 18:57:42,970] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_38_optim_states.pt -[default7]:[2022-09-11 18:57:42,984] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt. -[default7]:[2022-09-11 18:57:42,984] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_17_optim_states.pt -[default0]:[2022-09-11 18:57:42,926] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt. -[default0]:[2022-09-11 18:57:42,926] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_38_optim_states.pt -[default6]:[2022-09-11 18:57:43,414] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt. -[default6]:[2022-09-11 18:57:43,415] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_17_optim_states.pt -[default3]:[2022-09-11 18:57:43,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt. -[default3]:[2022-09-11 18:57:43,595] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_70_optim_states.pt -[default2]:[2022-09-11 18:57:43,787] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt. -[default2]:[2022-09-11 18:57:43,787] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_70_optim_states.pt -[default0]:[2022-09-11 18:57:43,981] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt. -[default0]:[2022-09-11 18:57:43,981] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_70_optim_states.pt -[default1]:[2022-09-11 18:57:43,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt. -[default1]:[2022-09-11 18:57:43,995] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_70_optim_states.pt -[default3]:[2022-09-11 18:57:44,393] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt. -[default3]:[2022-09-11 18:57:44,393] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_50_optim_states.pt -[default1]:[2022-09-11 18:57:44,481] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt. -[default1]:[2022-09-11 18:57:44,482] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_50_optim_states.pt -[default2]:[2022-09-11 18:57:44,457] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt. -[default2]:[2022-09-11 18:57:44,457] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_50_optim_states.pt -[default4]:[2022-09-11 18:57:44,501] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt. -[default4]:[2022-09-11 18:57:44,502] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_51_optim_states.pt -[default0]:[2022-09-11 18:57:44,527] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt. -[default0]:[2022-09-11 18:57:44,527] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_50_optim_states.pt -[default2]:[2022-09-11 18:57:44,531] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. -[default2]:[2022-09-11 18:57:44,531] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt -[default3]:[2022-09-11 18:57:44,680] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. -[default3]:[2022-09-11 18:57:44,681] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt -[default7]:[2022-09-11 18:57:44,896] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt. -[default7]:[2022-09-11 18:57:44,897] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_51_optim_states.pt -[default6]:[2022-09-11 18:57:44,998] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt. -[default6]:[2022-09-11 18:57:44,999] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_51_optim_states.pt -[default5]:[2022-09-11 18:57:45,434] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt. -[default5]:[2022-09-11 18:57:45,434] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_51_optim_states.pt -[default0]:[2022-09-11 18:57:46,715] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. -[default0]:[2022-09-11 18:57:46,719] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt -[default1]:[2022-09-11 18:57:46,787] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. -[default1]:[2022-09-11 18:57:46,787] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt -[default6]:[2022-09-11 18:57:51,360] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt. -[default6]:[2022-09-11 18:57:51,360] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_2_mp_rank_71_optim_states.pt -[default7]:[2022-09-11 18:57:51,432] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt. -[default7]:[2022-09-11 18:57:51,432] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_3_mp_rank_71_optim_states.pt -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:time (ms) | save-checkpoint: 39571.09 -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,778] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt. -[default5]:[2022-09-11 18:57:54,778] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_1_mp_rank_71_optim_states.pt -[default5]:[2022-09-11 18:57:54,778] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,754] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt. -[default4]:[2022-09-11 18:57:54,754] [INFO] [engine.py:3188:_save_zero_checkpoint] bf16_zero checkpoint saved /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq/global_step2357/bf16_zero_pp_rank_0_mp_rank_71_optim_states.pt -[default4]:[2022-09-11 18:57:54,778] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]: successfully saved checkpoint at iteration 2357 to /gpfsscratch/rech/six/commun/checkpoints/tr13-176B-ml-t0/checkpoints/xp3zzlossseq -[default0]:[Detected kill switch at /gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0/megdslossseqnew/Megatron-DeepSpeed/kill-switch-tr13-176B-mtf. Exiting] datetime: 2022-09-11 18:57:54 -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default5]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default1]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default2]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default4]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default3]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default6]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default7]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now! -[default0]:[2022-09-11 18:57:54,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2357 is ready now!